import asyncio import logging from typing import List, Optional from sqlalchemy import orm from web3 import Web3 from mev_inspect.fees import fetch_base_fee_per_gas from mev_inspect.schemas.blocks import Block from mev_inspect.schemas.liquidations import Liquidation from mev_inspect.schemas.receipts import Receipt from mev_inspect.schemas.swaps import Swap from mev_inspect.schemas.traces import Trace, TraceType logger = logging.getLogger(__name__) async def get_latest_block_number(w3) -> int: return await w3.eth.get_block_number() async def _get_logs_for_topics(w3, after_block, before_block, topics): return await w3.eth.get_logs( { "fromBlock": hex(after_block), "toBlock": hex(before_block), "topics": topics, } ) def twos_complement(hexstr, bits): value = int(hexstr, 16) if value & (1 << (bits - 1)): value -= 1 << bits return value def get_swap_v2(data): data = data[2:] # print(data) return ( int(data[0:64], base=16), int(data[64:128], base=16), int(data[128:192], base=16), int(data[192:256], base=16), ) def get_swap_v3(data): data = data[2:] # print(data) return ( twos_complement(data[0:64], 256), twos_complement(data[64:128], 256), int(data[152:192], base=16), int(data[224:256], base=16), twos_complement(data[314:320], 24), ) def get_swap_synapse(data): data = data[2:] # print(data) return ( int(data[0:64], base=16), int(data[64:128], base=16), data[128:192], data[192:256], ) def get_liquidation(data): data = data[2:] # print(data) return ( int(data[0:64], base=16), int(data[64:128], base=16), "0x" + data[128 + 24 : 168 + 24], ) univ2abi = """ [{"inputs":[],"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"owner","type":"address"},{"indexed":true,"internalType":"address","name":"spender","type":"address"},{"indexed":false,"internalType":"uint256","name":"value","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"sender","type":"address"},{"indexed":false,"internalType":"uint256","name":"amount0","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1","type":"uint256"},{"indexed":true,"internalType":"address","name":"to","type":"address"}],"name":"Burn","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"sender","type":"address"},{"indexed":false,"internalType":"uint256","name":"amount0","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1","type":"uint256"}],"name":"Mint","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"sender","type":"address"},{"indexed":false,"internalType":"uint256","name":"amount0In","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1In","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount0Out","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1Out","type":"uint256"},{"indexed":true,"internalType":"address","name":"to","type":"address"}],"name":"Swap","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"uint112","name":"reserve0","type":"uint112"},{"indexed":false,"internalType":"uint112","name":"reserve1","type":"uint112"}],"name":"Sync","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"from","type":"address"},{"indexed":true,"internalType":"address","name":"to","type":"address"},{"indexed":false,"internalType":"uint256","name":"value","type":"uint256"}],"name":"Transfer","type":"event"},{"inputs":[],"name":"DOMAIN_SEPARATOR","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"MINIMUM_LIQUIDITY","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"PERMIT_TYPEHASH","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"}],"name":"allowance","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"spender","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"}],"name":"approve","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"}],"name":"balanceOf","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"}],"name":"burn","outputs":[{"internalType":"uint256","name":"amount0","type":"uint256"},{"internalType":"uint256","name":"amount1","type":"uint256"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"decimals","outputs":[{"internalType":"uint8","name":"","type":"uint8"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"factory","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"getReserves","outputs":[{"internalType":"uint112","name":"_reserve0","type":"uint112"},{"internalType":"uint112","name":"_reserve1","type":"uint112"},{"internalType":"uint32","name":"_blockTimestampLast","type":"uint32"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_token0","type":"address"},{"internalType":"address","name":"_token1","type":"address"}],"name":"initialize","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"kLast","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"}],"name":"mint","outputs":[{"internalType":"uint256","name":"liquidity","type":"uint256"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"name","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"}],"name":"nonces","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"owner","type":"address"},{"internalType":"address","name":"spender","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"},{"internalType":"uint256","name":"deadline","type":"uint256"},{"internalType":"uint8","name":"v","type":"uint8"},{"internalType":"bytes32","name":"r","type":"bytes32"},{"internalType":"bytes32","name":"s","type":"bytes32"}],"name":"permit","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"price0CumulativeLast","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"price1CumulativeLast","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"}],"name":"skim","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"amount0Out","type":"uint256"},{"internalType":"uint256","name":"amount1Out","type":"uint256"},{"internalType":"address","name":"to","type":"address"},{"internalType":"bytes","name":"data","type":"bytes"}],"name":"swap","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"symbol","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"sync","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"token0","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"token1","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"totalSupply","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"}],"name":"transfer","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"from","type":"address"},{"internalType":"address","name":"to","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"}],"name":"transferFrom","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"}] """ def parse_topic(log, index): return log["topics"][index].hex() def parse_blockNumber(log): return log["blockNumber"] def parse_transactionHash(log): return log["transactionHash"].hex() def parse_address(log): return log["address"].lower() def parse_logIndex(log): return log["logIndex"] def parse_data(log): return log["data"] def parse_token(w3, token): return "0x" + w3.toHex(token)[26:] async def classify_logs(logs, reserves, w3): cswaps = [] cliquidations = [] new_reserves = [] topic_swap_v2 = "0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822" topic_swap_v3 = "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67" topic_swap_synapse = ( "0xc6c1e0630dbe9130cc068028486c0d118ddcea348550819defd5cb8c257f8a38" ) topic_liquidation = ( "0xe413a321e8681d831f4dbccbca790d2952b56f977908e45be37335533e005286" ) for log in logs: if parse_topic(log, 0) == topic_swap_v2: block = parse_blockNumber(log) transaction_hash = parse_transactionHash(log) pool_address = parse_address(log) if pool_address in reserves: token0, token1 = reserves[pool_address] else: addr = Web3.toChecksumAddress(pool_address) token0, token1 = await asyncio.gather( w3.eth.call({"to": addr, "data": "0x0dfe1681"}), w3.eth.call({"to": addr, "data": "0xd21220a7"}), ) token0 = parse_token(w3, token0) token1 = parse_token(w3, token1) reserves[pool_address] = (token0, token1) new_reserves.append( {"pool_address": pool_address, "token0": token0, "token1": token1} ) am0in, am1in, am0out, am1out = get_swap_v2(parse_data(log)) swap = Swap( abi_name="uniswap_v2", transaction_hash=transaction_hash, block_number=block, trace_address=[parse_logIndex(log)], contract_address=pool_address, from_address="0x" + parse_topic(log, 1)[26:], to_address="0x" + parse_topic(log, 2)[26:], token_in_address=token0 if am0in != 0 else token1, # TODO token_in_amount=am0in if am0in != 0 else am1in, token_out_address=token1 if am1out != 0 else token0, # TODO token_out_amount=am0out if am0out != 0 else am1out, protocol=None, error=None, ) cswaps.append(swap) elif parse_topic(log, 0) == topic_swap_v3: block = parse_blockNumber(log) transaction_hash = parse_transactionHash(log) pool_address = parse_address(log) if pool_address in reserves: token0, token1 = reserves[pool_address] else: addr = Web3.toChecksumAddress(pool_address) token0, token1 = await asyncio.gather( w3.eth.call({"to": addr, "data": "0x0dfe1681"}), w3.eth.call({"to": addr, "data": "0xd21220a7"}), ) token0 = parse_token(w3, token0) token1 = parse_token(w3, token1) reserves[pool_address] = (token0, token1) new_reserves.append( {"pool_address": pool_address, "token0": token0, "token1": token1} ) am0, am1, _, _, _ = get_swap_v3(parse_data(log)) swap = Swap( abi_name="uniswap_v3", transaction_hash=transaction_hash, block_number=block, trace_address=[parse_logIndex(log)], contract_address=pool_address, from_address="0x" + parse_topic(log, 1)[26:], to_address="0x" + parse_topic(log, 2)[26:], token_in_address=token0 if am0 > 0 else token1, # TODO token_in_amount=am0 if am0 > 0 else am1, token_out_address=token1 if am1 < 0 else token0, # TODO token_out_amount=-am1 if am1 < 0 else -am0, protocol=None, error=None, ) cswaps.append(swap) elif parse_topic(log, 0) == topic_swap_synapse: block = parse_blockNumber(log) transaction_hash = parse_transactionHash(log) pool_address = parse_address(log) sold, bought, soldId, boughtId = get_swap_synapse(parse_data(log)) if pool_address in reserves: pass # TODO else: addr = Web3.toChecksumAddress(pool_address) soldToken, boughtToken = await asyncio.gather( w3.eth.call({"to": addr, "data": "0x82b86600" + soldId}), w3.eth.call({"to": addr, "data": "0x82b86600" + boughtId}), ) soldToken = parse_token(w3, soldToken) boughtToken = parse_token(w3, boughtToken) # reserves[pool_address] = (token0, token1) # new_reserves.append( # { # "pool_address": pool_address, # "token0": token0, # "token1": token1 # } # ) swap = Swap( abi_name="synapse", transaction_hash=transaction_hash, block_number=block, trace_address=[parse_logIndex(log)], contract_address=pool_address, from_address="0x" + parse_topic(log, 1)[26:], to_address="0x" + parse_topic(log, 1)[26:], token_in_address=soldToken, token_in_amount=sold, token_out_address=boughtToken, token_out_amount=bought, protocol=None, error=None, ) cswaps.append(swap) elif parse_topic(log, 0) == topic_liquidation: block = str(parse_blockNumber(log)) am_debt, am_recv, addr_usr = get_liquidation(parse_data(log)) liquidation = Liquidation( liquidated_user="0x" + parse_topic(log, 3)[26:], liquidator_user=addr_usr, debt_token_address="0x" + parse_topic(log, 2)[26:], debt_purchase_amount=am_debt, received_amount=am_recv, received_token_address="0x" + parse_topic(log, 1)[26:], protocol=None, transaction_hash=parse_transactionHash(log), trace_address=[parse_logIndex(log)], block_number=block, error=None, ) cliquidations.append(liquidation) return cswaps, cliquidations, new_reserves async def get_classified_traces_from_events(w3, after_block, before_block, reserves): start = after_block stride = 2000 topic_swap_v2 = "0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822" topic_swap_v3 = "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67" topic_swap_synapse = ( "0xc6c1e0630dbe9130cc068028486c0d118ddcea348550819defd5cb8c257f8a38" ) topic_liquidation = ( "0xe413a321e8681d831f4dbccbca790d2952b56f977908e45be37335533e005286" ) while start < before_block: begin = start end = start + stride if (start + stride) < before_block else before_block end -= 1 start += stride logger.info("fetching from node... {} {}".format(begin, end)) all_logs = await _get_logs_for_topics( w3, begin, end, [[topic_swap_v2, topic_swap_v3, topic_swap_synapse, topic_liquidation]], ) yield await classify_logs(all_logs, reserves, w3) async def create_from_block_number( w3: Web3, block_number: int, trace_db_session: Optional[orm.Session], ) -> Block: block_timestamp, receipts, traces, base_fee_per_gas = await asyncio.gather( _find_or_fetch_block_timestamp(w3, block_number, trace_db_session), _find_or_fetch_block_receipts(w3, block_number, trace_db_session), _find_or_fetch_block_traces(w3, block_number, trace_db_session), _find_or_fetch_base_fee_per_gas(w3, block_number, trace_db_session), ) miner_address = _get_miner_address_from_traces(traces) return Block( block_number=block_number, block_timestamp=block_timestamp, miner=miner_address, base_fee_per_gas=base_fee_per_gas, traces=traces, receipts=receipts, ) async def _find_or_fetch_block_timestamp( w3, block_number: int, trace_db_session: Optional[orm.Session], ) -> int: if trace_db_session is not None: existing_block_timestamp = _find_block_timestamp(trace_db_session, block_number) if existing_block_timestamp is not None: return existing_block_timestamp return await _fetch_block_timestamp(w3, block_number) async def _find_or_fetch_block_receipts( w3, block_number: int, trace_db_session: Optional[orm.Session], ) -> List[Receipt]: if trace_db_session is not None: existing_block_receipts = _find_block_receipts(trace_db_session, block_number) if existing_block_receipts is not None: return existing_block_receipts return await _fetch_block_receipts(w3, block_number) async def _find_or_fetch_block_traces( w3, block_number: int, trace_db_session: Optional[orm.Session], ) -> List[Trace]: if trace_db_session is not None: existing_block_traces = _find_block_traces(trace_db_session, block_number) if existing_block_traces is not None: return existing_block_traces return await _fetch_block_traces(w3, block_number) async def _find_or_fetch_base_fee_per_gas( w3, block_number: int, trace_db_session: Optional[orm.Session], ) -> int: if trace_db_session is not None: existing_base_fee_per_gas = _find_base_fee_per_gas( trace_db_session, block_number ) if existing_base_fee_per_gas is not None: return existing_base_fee_per_gas return await fetch_base_fee_per_gas(w3, block_number) async def _fetch_block_timestamp(w3, block_number: int) -> int: block_json = await w3.eth.get_block(block_number) return block_json["timestamp"] async def _fetch_block_receipts(w3, block_number: int) -> List[Receipt]: receipts_json = await w3.eth.get_block_receipts(block_number) return [Receipt(**receipt) for receipt in receipts_json] async def _fetch_block_traces(w3, block_number: int) -> List[Trace]: traces_json = await w3.eth.trace_block(block_number) return [Trace(**trace_json) for trace_json in traces_json] def _find_block_timestamp( trace_db_session: orm.Session, block_number: int, ) -> Optional[int]: result = trace_db_session.execute( "SELECT block_timestamp FROM block_timestamps WHERE block_number = :block_number", params={"block_number": block_number}, ).one_or_none() if result is None: return None else: (block_timestamp,) = result return block_timestamp def _find_block_traces( trace_db_session: orm.Session, block_number: int, ) -> Optional[List[Trace]]: result = trace_db_session.execute( "SELECT raw_traces FROM block_traces WHERE block_number = :block_number", params={"block_number": block_number}, ).one_or_none() if result is None: return None else: (traces_json,) = result return [Trace(**trace_json) for trace_json in traces_json] def _find_block_receipts( trace_db_session: orm.Session, block_number: int, ) -> Optional[List[Receipt]]: result = trace_db_session.execute( "SELECT raw_receipts FROM block_receipts WHERE block_number = :block_number", params={"block_number": block_number}, ).one_or_none() if result is None: return None else: (receipts_json,) = result return [Receipt(**receipt) for receipt in receipts_json] def _find_base_fee_per_gas( trace_db_session: orm.Session, block_number: int, ) -> Optional[int]: result = trace_db_session.execute( "SELECT base_fee_in_wei FROM base_fee WHERE block_number = :block_number", params={"block_number": block_number}, ).one_or_none() if result is None: return None else: (base_fee,) = result return base_fee def _get_miner_address_from_traces(traces: List[Trace]) -> Optional[str]: for trace in traces: if trace.type == TraceType.reward: return trace.action["author"] return None def get_transaction_hashes(calls: List[Trace]) -> List[str]: result = [] for call in calls: if call.type != TraceType.reward: if ( call.transaction_hash is not None and call.transaction_hash not in result ): result.append(call.transaction_hash) return result