diff --git a/mev_inspect/block.py b/mev_inspect/block.py index cf80fa5..8b18078 100644 --- a/mev_inspect/block.py +++ b/mev_inspect/block.py @@ -1,5 +1,6 @@ import asyncio import logging +import time from typing import List, Optional from sqlalchemy import orm @@ -9,6 +10,8 @@ from mev_inspect.fees import fetch_base_fee_per_gas from mev_inspect.schemas.blocks import Block from mev_inspect.schemas.receipts import Receipt from mev_inspect.schemas.traces import Trace, TraceType +from mev_inspect.schemas.swaps import Swap + from mev_inspect.utils import hex_to_int logger = logging.getLogger(__name__) @@ -22,6 +25,90 @@ async def get_latest_block_number(base_provider) -> int: return hex_to_int(latest_block["result"]["number"]) +async def _get_logs_for_topics(base_provider, after_block, before_block, topics): + print("getting log traces") + start = time.time() + logs = await base_provider.make_request("eth_getLogs", + [{ + "fromBlock": hex(after_block), + "toBlock": hex(before_block), + "topics": topics, + }]) + print("getting log traces done ", time.time() - start, len(logs)) + return logs['result'] + +def _logs_by_tx(logs): + logs_by_tx = dict() + for log in logs: + transaction_hash = log['transactionHash'] + if transaction_hash in logs_by_tx.keys(): + logs_by_tx[transaction_hash].append(log) + else: + logs_by_tx[transaction_hash] = [log] + return logs_by_tx + +def get_amounts(data): + data = data[2:] + # print(data) + return int(data[0:64], base=16), int(data[64:128], base=16), int(data[128:192], base=16), int(data[192:256], base=16) + +univ2abi = ''' +[{"inputs":[],"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"owner","type":"address"},{"indexed":true,"internalType":"address","name":"spender","type":"address"},{"indexed":false,"internalType":"uint256","name":"value","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"sender","type":"address"},{"indexed":false,"internalType":"uint256","name":"amount0","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1","type":"uint256"},{"indexed":true,"internalType":"address","name":"to","type":"address"}],"name":"Burn","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"sender","type":"address"},{"indexed":false,"internalType":"uint256","name":"amount0","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1","type":"uint256"}],"name":"Mint","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"sender","type":"address"},{"indexed":false,"internalType":"uint256","name":"amount0In","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1In","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount0Out","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1Out","type":"uint256"},{"indexed":true,"internalType":"address","name":"to","type":"address"}],"name":"Swap","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"uint112","name":"reserve0","type":"uint112"},{"indexed":false,"internalType":"uint112","name":"reserve1","type":"uint112"}],"name":"Sync","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"from","type":"address"},{"indexed":true,"internalType":"address","name":"to","type":"address"},{"indexed":false,"internalType":"uint256","name":"value","type":"uint256"}],"name":"Transfer","type":"event"},{"inputs":[],"name":"DOMAIN_SEPARATOR","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"MINIMUM_LIQUIDITY","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"PERMIT_TYPEHASH","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"}],"name":"allowance","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"spender","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"}],"name":"approve","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"}],"name":"balanceOf","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"}],"name":"burn","outputs":[{"internalType":"uint256","name":"amount0","type":"uint256"},{"internalType":"uint256","name":"amount1","type":"uint256"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"decimals","outputs":[{"internalType":"uint8","name":"","type":"uint8"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"factory","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"getReserves","outputs":[{"internalType":"uint112","name":"_reserve0","type":"uint112"},{"internalType":"uint112","name":"_reserve1","type":"uint112"},{"internalType":"uint32","name":"_blockTimestampLast","type":"uint32"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_token0","type":"address"},{"internalType":"address","name":"_token1","type":"address"}],"name":"initialize","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"kLast","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"}],"name":"mint","outputs":[{"internalType":"uint256","name":"liquidity","type":"uint256"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"name","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"}],"name":"nonces","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"owner","type":"address"},{"internalType":"address","name":"spender","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"},{"internalType":"uint256","name":"deadline","type":"uint256"},{"internalType":"uint8","name":"v","type":"uint8"},{"internalType":"bytes32","name":"r","type":"bytes32"},{"internalType":"bytes32","name":"s","type":"bytes32"}],"name":"permit","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"price0CumulativeLast","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"price1CumulativeLast","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"}],"name":"skim","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"amount0Out","type":"uint256"},{"internalType":"uint256","name":"amount1Out","type":"uint256"},{"internalType":"address","name":"to","type":"address"},{"internalType":"bytes","name":"data","type":"bytes"}],"name":"swap","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"symbol","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"sync","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"token0","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"token1","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"totalSupply","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"}],"name":"transfer","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"from","type":"address"},{"internalType":"address","name":"to","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"}],"name":"transferFrom","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"}] +''' + +async def classify_logs(logs, reserves, w3): + cswaps = [] + topic_swap = "0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822" + for log in logs: + if log['topics'][0] == topic_swap: + # print(log) + block = int(log['blockNumber'], 16) + transaction_hash = log['transactionHash'] + pool_address = log['address'] + if pool_address in reserves: + token0, token1 = reserves[pool_address] + else: + addr = Web3.toChecksumAddress(pool_address) + token0 = await w3.eth.call({'to': addr, 'data': '0x0dfe1681'}) + token1 = await w3.eth.call({'to': addr, 'data': '0xd21220a7'}) + token0 = w3.toHex(token0) + token1 = w3.toHex(token1) + reserves[pool_address] = (token0, token1) + + am0in, am1in, am0out, am1out = get_amounts(log['data']) + swap = Swap( + abi_name="uniswap_v2", + transaction_hash=transaction_hash, + block_number=block, + trace_address=[int(log['logIndex'], 16)], + contract_address=pool_address, + from_address="0x"+log['topics'][1][26:], + to_address="0x"+log['topics'][2][26:], + token_in_address=token0 if am0in != 0 else token1, # TODO + token_in_amount= am0in if am0in != 0 else am1in, + token_out_address=token1 if am1out != 0 else token0, # TODO + token_out_amount= am0out if am0out != 0 else am1out, + protocol=None, + error=None + ) + cswaps.append(swap) + + return cswaps + +async def get_classified_traces_from_events(w3: Web3, after_block: int, before_block: int): + base_provider = w3.provider + start = after_block + stride = 300 + reserves = dict() + while start < before_block: + begin = start + end = start + stride if (start + stride) < before_block else before_block + start += stride + print("fetching from node...", begin, end, flush=True) + swaplogs = await _get_logs_for_topics(base_provider, begin, end, ["0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822"]) + logs_by_tx = _logs_by_tx(swaplogs) + for tx in logs_by_tx.keys(): + yield await classify_logs(logs_by_tx[tx], reserves, w3) async def create_from_block_number( w3: Web3, diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py index db24347..9680758 100644 --- a/mev_inspect/inspect_block.py +++ b/mev_inspect/inspect_block.py @@ -5,7 +5,7 @@ from sqlalchemy import orm from web3 import Web3 from mev_inspect.arbitrages import get_arbitrages -from mev_inspect.block import create_from_block_number +from mev_inspect.block import create_from_block_number, get_classified_traces_from_events from mev_inspect.classifiers.trace import TraceClassifier from mev_inspect.crud.arbitrages import delete_arbitrages_for_blocks, write_arbitrages from mev_inspect.crud.blocks import delete_blocks, write_blocks @@ -85,152 +85,177 @@ async def inspect_many_blocks( trace_db_session: Optional[orm.Session], should_write_classified_traces: bool = True, ): - all_blocks: List[Block] = [] - all_classified_traces: List[ClassifiedTrace] = [] - all_transfers: List[Transfer] = [] - all_swaps: List[Swap] = [] - all_arbitrages: List[Arbitrage] = [] - all_liquidations: List[Liquidation] = [] - all_sandwiches: List[Sandwich] = [] - - all_punk_bids: List[PunkBid] = [] - all_punk_bid_acceptances: List[PunkBidAcceptance] = [] - all_punk_snipes: List[PunkSnipe] = [] - - all_miner_payments: List[MinerPayment] = [] - - all_nft_trades: List[NftTrade] = [] - - for block_number in range(after_block_number, before_block_number): - block = await create_from_block_number( - w3, - block_number, - trace_db_session, - ) - - logger.info(f"Block: {block_number} -- Total traces: {len(block.traces)}") - - total_transactions = len( - set( - t.transaction_hash - for t in block.traces - if t.transaction_hash is not None - ) - ) - logger.info( - f"Block: {block_number} -- Total transactions: {total_transactions}" - ) - - classified_traces = trace_classifier.classify(block.traces) - logger.info( - f"Block: {block_number} -- Returned {len(classified_traces)} classified traces" - ) - - transfers = get_transfers(classified_traces) - logger.info(f"Block: {block_number} -- Found {len(transfers)} transfers") - - swaps = get_swaps(classified_traces) - logger.info(f"Block: {block_number} -- Found {len(swaps)} swaps") + count = 0 + arbitrages_payload = [] + async for swaps in get_classified_traces_from_events(w3, after_block_number, before_block_number): arbitrages = get_arbitrages(swaps) - logger.info(f"Block: {block_number} -- Found {len(arbitrages)} arbitrages") + count += len(arbitrages) + logger.info(f"{count} Found {len(swaps)} swaps and {len(arbitrages)} arbitrages") + if len(arbitrages) > 0: + for arb in arbitrages: + arb_payload = dict() + arb_payload['block_number'] = arb.block_number + arb_payload['transaction'] = arb.transaction_hash + arb_payload['account'] = arb.account_address + arb_payload['profit_amt'] = arb.profit_amount + arb_payload['token'] = arb.profit_token_address + arbitrages_payload.append(arb_payload) + count += 1 + + if count >= 100: + print("sending to endpoint now") + # resp = requests.post("https://asia-south1-marlin-internal.cloudfunctions.net/mevPolygon/alerts", headers={'Content-type': 'application/json'}, json={"arbitrages": arbitrages_payload}) + # print("sending to endpoint ", resp.content.decode("utf-8"), flush=True) + arbitrages_payload = [] + count = 0 - liquidations = get_liquidations(classified_traces) - logger.info(f"Block: {block_number} -- Found {len(liquidations)} liquidations") + # all_blocks: List[Block] = [] + # all_classified_traces: List[ClassifiedTrace] = [] + # all_transfers: List[Transfer] = [] + # all_swaps: List[Swap] = [] + # all_arbitrages: List[Arbitrage] = [] + # all_liquidations: List[Liquidation] = [] + # all_sandwiches: List[Sandwich] = [] - sandwiches = get_sandwiches(swaps) - logger.info(f"Block: {block_number} -- Found {len(sandwiches)} sandwiches") + # all_punk_bids: List[PunkBid] = [] + # all_punk_bid_acceptances: List[PunkBidAcceptance] = [] + # all_punk_snipes: List[PunkSnipe] = [] - punk_bids = get_punk_bids(classified_traces) - punk_bid_acceptances = get_punk_bid_acceptances(classified_traces) - punk_snipes = get_punk_snipes(punk_bids, punk_bid_acceptances) - logger.info(f"Block: {block_number} -- Found {len(punk_snipes)} punk snipes") + # all_miner_payments: List[MinerPayment] = [] - nft_trades = get_nft_trades(classified_traces) - logger.info(f"Block: {block_number} -- Found {len(nft_trades)} nft trades") + # all_nft_trades: List[NftTrade] = [] - miner_payments = get_miner_payments( - block.miner, block.base_fee_per_gas, classified_traces, block.receipts - ) + # for block_number in range(after_block_number, before_block_number): + # block = await create_from_block_number( + # w3, + # block_number, + # trace_db_session, + # ) - all_blocks.append(block) - all_classified_traces.extend(classified_traces) - all_transfers.extend(transfers) - all_swaps.extend(swaps) - all_arbitrages.extend(arbitrages) - all_liquidations.extend(liquidations) - all_sandwiches.extend(sandwiches) + # logger.info(f"Block: {block_number} -- Total traces: {len(block.traces)}") - all_punk_bids.extend(punk_bids) - all_punk_bid_acceptances.extend(punk_bid_acceptances) - all_punk_snipes.extend(punk_snipes) + # total_transactions = len( + # set( + # t.transaction_hash + # for t in block.traces + # if t.transaction_hash is not None + # ) + # ) + # logger.info( + # f"Block: {block_number} -- Total transactions: {total_transactions}" + # ) - all_nft_trades.extend(nft_trades) + # classified_traces = trace_classifier.classify(block.traces) + # logger.info( + # f"Block: {block_number} -- Returned {len(classified_traces)} classified traces" + # ) - all_miner_payments.extend(miner_payments) + # transfers = get_transfers(classified_traces) + # logger.info(f"Block: {block_number} -- Found {len(transfers)} transfers") - logger.info("Writing data") - delete_blocks(inspect_db_session, after_block_number, before_block_number) - write_blocks(inspect_db_session, all_blocks) + # swaps = get_swaps(classified_traces) + # logger.info(f"Block: {block_number} -- Found {len(swaps)} swaps") - if should_write_classified_traces: - delete_classified_traces_for_blocks( - inspect_db_session, after_block_number, before_block_number - ) - write_classified_traces(inspect_db_session, all_classified_traces) + # arbitrages = get_arbitrages(swaps) + # logger.info(f"Block: {block_number} -- Found {len(arbitrages)} arbitrages") - delete_transfers_for_blocks( - inspect_db_session, after_block_number, before_block_number - ) - write_transfers(inspect_db_session, all_transfers) + # liquidations = get_liquidations(classified_traces) + # logger.info(f"Block: {block_number} -- Found {len(liquidations)} liquidations") - delete_swaps_for_blocks(inspect_db_session, after_block_number, before_block_number) - write_swaps(inspect_db_session, all_swaps) + # sandwiches = get_sandwiches(swaps) + # logger.info(f"Block: {block_number} -- Found {len(sandwiches)} sandwiches") - delete_arbitrages_for_blocks( - inspect_db_session, after_block_number, before_block_number - ) - write_arbitrages(inspect_db_session, all_arbitrages) + # punk_bids = get_punk_bids(classified_traces) + # punk_bid_acceptances = get_punk_bid_acceptances(classified_traces) + # punk_snipes = get_punk_snipes(punk_bids, punk_bid_acceptances) + # logger.info(f"Block: {block_number} -- Found {len(punk_snipes)} punk snipes") - delete_liquidations_for_blocks( - inspect_db_session, after_block_number, before_block_number - ) - write_liquidations(inspect_db_session, all_liquidations) + # nft_trades = get_nft_trades(classified_traces) + # logger.info(f"Block: {block_number} -- Found {len(nft_trades)} nft trades") - delete_sandwiches_for_blocks( - inspect_db_session, after_block_number, before_block_number - ) - write_sandwiches(inspect_db_session, all_sandwiches) + # miner_payments = get_miner_payments( + # block.miner, block.base_fee_per_gas, classified_traces, block.receipts + # ) - delete_punk_bids_for_blocks( - inspect_db_session, after_block_number, before_block_number - ) - write_punk_bids(inspect_db_session, all_punk_bids) + # all_blocks.append(block) + # all_classified_traces.extend(classified_traces) + # all_transfers.extend(transfers) + # all_swaps.extend(swaps) + # all_arbitrages.extend(arbitrages) + # all_liquidations.extend(liquidations) + # all_sandwiches.extend(sandwiches) - delete_punk_bid_acceptances_for_blocks( - inspect_db_session, after_block_number, before_block_number - ) - write_punk_bid_acceptances(inspect_db_session, all_punk_bid_acceptances) + # all_punk_bids.extend(punk_bids) + # all_punk_bid_acceptances.extend(punk_bid_acceptances) + # all_punk_snipes.extend(punk_snipes) - delete_punk_snipes_for_blocks( - inspect_db_session, after_block_number, before_block_number - ) - write_punk_snipes(inspect_db_session, all_punk_snipes) + # all_nft_trades.extend(nft_trades) - delete_nft_trades_for_blocks( - inspect_db_session, after_block_number, before_block_number - ) - write_nft_trades(inspect_db_session, all_nft_trades) + # all_miner_payments.extend(miner_payments) - delete_miner_payments_for_blocks( - inspect_db_session, after_block_number, before_block_number - ) - write_miner_payments(inspect_db_session, all_miner_payments) + # logger.info("Writing data") + # delete_blocks(inspect_db_session, after_block_number, before_block_number) + # write_blocks(inspect_db_session, all_blocks) - update_summary_for_block_range( - inspect_db_session, - after_block_number, - before_block_number, - ) + # if should_write_classified_traces: + # delete_classified_traces_for_blocks( + # inspect_db_session, after_block_number, before_block_number + # ) + # write_classified_traces(inspect_db_session, all_classified_traces) - logger.info("Done writing") + # delete_transfers_for_blocks( + # inspect_db_session, after_block_number, before_block_number + # ) + # write_transfers(inspect_db_session, all_transfers) + + # delete_swaps_for_blocks(inspect_db_session, after_block_number, before_block_number) + # write_swaps(inspect_db_session, all_swaps) + + # delete_arbitrages_for_blocks( + # inspect_db_session, after_block_number, before_block_number + # ) + # write_arbitrages(inspect_db_session, all_arbitrages) + + # delete_liquidations_for_blocks( + # inspect_db_session, after_block_number, before_block_number + # ) + # write_liquidations(inspect_db_session, all_liquidations) + + # delete_sandwiches_for_blocks( + # inspect_db_session, after_block_number, before_block_number + # ) + # write_sandwiches(inspect_db_session, all_sandwiches) + + # delete_punk_bids_for_blocks( + # inspect_db_session, after_block_number, before_block_number + # ) + # write_punk_bids(inspect_db_session, all_punk_bids) + + # delete_punk_bid_acceptances_for_blocks( + # inspect_db_session, after_block_number, before_block_number + # ) + # write_punk_bid_acceptances(inspect_db_session, all_punk_bid_acceptances) + + # delete_punk_snipes_for_blocks( + # inspect_db_session, after_block_number, before_block_number + # ) + # write_punk_snipes(inspect_db_session, all_punk_snipes) + + # delete_nft_trades_for_blocks( + # inspect_db_session, after_block_number, before_block_number + # ) + # write_nft_trades(inspect_db_session, all_nft_trades) + + # delete_miner_payments_for_blocks( + # inspect_db_session, after_block_number, before_block_number + # ) + # write_miner_payments(inspect_db_session, all_miner_payments) + + # update_summary_for_block_range( + # inspect_db_session, + # after_block_number, + # before_block_number, + # ) + + # logger.info("Done writing") diff --git a/mev_inspect/schemas/swaps.py b/mev_inspect/schemas/swaps.py index f15bc20..eb3fb25 100644 --- a/mev_inspect/schemas/swaps.py +++ b/mev_inspect/schemas/swaps.py @@ -8,7 +8,7 @@ from mev_inspect.schemas.traces import Protocol class Swap(BaseModel): abi_name: str transaction_hash: str - transaction_position: int + transaction_position: Optional[int] block_number: int trace_address: List[int] contract_address: str @@ -18,5 +18,5 @@ class Swap(BaseModel): token_in_amount: int token_out_address: str token_out_amount: int - protocol: Protocol + protocol: Optional[Protocol] error: Optional[str]