From 8504ac5ccaec2b67a60272d9fa4866c5894fef54 Mon Sep 17 00:00:00 2001 From: Supragya Raj Date: Thu, 25 Nov 2021 10:46:00 +0100 Subject: [PATCH] updates for latest master --- cli.py | 11 +++++------ mev_inspect/block.py | 32 +++++++++++++++++++++++--------- mev_inspect/inspect_block.py | 2 +- mev_inspect/inspector.py | 17 ++++++++++++++++- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/cli.py b/cli.py index 4206570..86f9594 100644 --- a/cli.py +++ b/cli.py @@ -3,8 +3,6 @@ import os import sys import click -from web3 import Web3 -from web3.middleware import geth_poa_middleware from mev_inspect.concurrency import coro from mev_inspect.db import get_inspect_session, get_trace_session @@ -26,6 +24,7 @@ def cli(): @click.option("--geth/--no-geth", default=False) @coro async def inspect_block_command(block_number: int, rpc: str, geth: bool): + print("geth", geth) inspect_db_session = get_inspect_session() trace_db_session = get_trace_session() @@ -41,7 +40,7 @@ async def fetch_block_command(block_number: int, rpc: str): inspect_db_session = get_inspect_session() trace_db_session = get_trace_session() - inspector = MEVInspector(rpc, inspect_db_session, trace_db_session, false) + inspector = MEVInspector(rpc, inspect_db_session, trace_db_session, False) block = await inspector.create_from_block(block_number=block_number) print(block.json()) @@ -51,7 +50,6 @@ async def fetch_block_command(block_number: int, rpc: str): @click.argument("before_block", type=int) @click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, "")) @click.option("--geth/--no-geth", default=False) - @click.option( "--max-concurrency", type=int, @@ -68,7 +66,7 @@ async def inspect_many_blocks_command( rpc: str, max_concurrency: int, request_timeout: int, - geth: bool + geth: bool, ): inspect_db_session = get_inspect_session() trace_db_session = get_trace_session() @@ -76,14 +74,15 @@ async def inspect_many_blocks_command( rpc, inspect_db_session, trace_db_session, + geth, max_concurrency=max_concurrency, request_timeout=request_timeout, - geth ) await inspector.inspect_many_blocks( after_block=after_block, before_block=before_block ) + def get_rpc_url() -> str: return os.environ["RPC_URL"] diff --git a/mev_inspect/block.py b/mev_inspect/block.py index 592600c..b207187 100644 --- a/mev_inspect/block.py +++ b/mev_inspect/block.py @@ -2,7 +2,6 @@ import asyncio import logging from typing import List, Optional import json -import asyncio import aiohttp from sqlalchemy import orm @@ -40,13 +39,14 @@ async def create_from_block_number( block = _find_block(trace_db_session, block_number) if block is None: - block = await _fetch_block(w3, base_provider, block_number) - return block - else: + block = await _fetch_block(w3, base_provider, geth, block_number) return block + return block -async def _fetch_block(w3, base_provider, geth, block_number: int, retries: int = 0) -> Block: +async def _fetch_block( + w3, base_provider, geth: bool, block_number: int, retries: int = 0 +) -> Block: if not geth: block_json, receipts_json, traces_json, base_fee_per_gas = await asyncio.gather( w3.eth.get_block(block_number), @@ -60,23 +60,35 @@ async def _fetch_block(w3, base_provider, geth, block_number: int, retries: int Receipt(**receipt) for receipt in receipts_json["result"] ] traces = [Trace(**trace_json) for trace_json in traces_json["result"]] + return Block( + block_number=block_number, + block_timestamp=block_json["timestamp"], + miner=block_json["miner"], + base_fee_per_gas=base_fee_per_gas, + traces=traces, + receipts=receipts, + ) except KeyError as e: logger.warning( f"Failed to create objects from block: {block_number}: {e}, retrying: {retries + 1} / 3" ) if retries < 3: await asyncio.sleep(5) - return await _fetch_block(w3, base_provider, block_number, retries) + return await _fetch_block( + w3, base_provider, geth, block_number, retries + ) else: raise else: + block_json = await asyncio.gather(w3.eth.get_block(block_number)) + print(block_json) traces = geth_get_tx_traces_parity_format(base_provider, block_json) geth_tx_receipts = geth_get_tx_receipts( base_provider, block_json["transactions"] ) receipts = geth_receipts_translator(block_json, geth_tx_receipts) base_fee_per_gas = 0 - + return Block( block_number=block_number, block_timestamp=block_json["timestamp"], @@ -85,7 +97,7 @@ async def _fetch_block(w3, base_provider, geth, block_number: int, retries: int traces=traces, receipts=receipts, ) - + def _find_block( trace_db_session: orm.Session, @@ -118,6 +130,7 @@ def _find_block( receipts=receipts, ) + def _find_block_timestamp( trace_db_session: orm.Session, block_number: int, @@ -203,10 +216,11 @@ def get_transaction_hashes(calls: List[Trace]) -> List[str]: return result + # Geth specific additions -def geth_get_tx_traces_parity_format(base_provider, block_json): +def geth_get_tx_traces_parity_format(base_provider, block_json: dict): block_hash = block_json["hash"] block_trace = geth_get_tx_traces(base_provider, block_hash) parity_traces = [] diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py index 572394a..53ad8a4 100644 --- a/mev_inspect/inspect_block.py +++ b/mev_inspect/inspect_block.py @@ -44,7 +44,7 @@ async def inspect_block( base_provider, w3: Web3, geth: bool, - trace_clasifier: TraceClassifier, + trace_classifier: TraceClassifier, block_number: int, trace_db_session: Optional[orm.Session], should_write_classified_traces: bool = True, diff --git a/mev_inspect/inspector.py b/mev_inspect/inspector.py index 4fb8160..0fe97f6 100644 --- a/mev_inspect/inspector.py +++ b/mev_inspect/inspector.py @@ -7,6 +7,7 @@ from typing import Optional from sqlalchemy import orm from web3 import Web3 from web3.eth import AsyncEth +from web3.middleware import geth_poa_middleware from mev_inspect.block import create_from_block_number from mev_inspect.classifiers.trace import TraceClassifier @@ -22,13 +23,24 @@ class MEVInspector: rpc: str, inspect_db_session: orm.Session, trace_db_session: Optional[orm.Session], + geth: bool = False, max_concurrency: int = 1, request_timeout: int = 300, ): self.inspect_db_session = inspect_db_session self.trace_db_session = trace_db_session self.base_provider = get_base_provider(rpc, request_timeout=request_timeout) - self.w3 = Web3(self.base_provider, modules={"eth": (AsyncEth,)}, middlewares=[]) + self.geth = geth + if geth: + self.w3 = Web3( + self.base_provider, + modules={"eth": (AsyncEth,)}, + middlewares=[geth_poa_middleware], + ) + else: + self.w3 = Web3( + self.base_provider, modules={"eth": (AsyncEth,)}, middlewares=[] + ) self.trace_classifier = TraceClassifier() self.max_concurrency = asyncio.Semaphore(max_concurrency) @@ -36,6 +48,7 @@ class MEVInspector: return await create_from_block_number( base_provider=self.base_provider, w3=self.w3, + geth=self.geth, block_number=block_number, trace_db_session=self.trace_db_session, ) @@ -45,6 +58,7 @@ class MEVInspector: self.inspect_db_session, self.base_provider, self.w3, + self.geth, self.trace_classifier, block, trace_db_session=self.trace_db_session, @@ -73,6 +87,7 @@ class MEVInspector: self.inspect_db_session, self.base_provider, self.w3, + self.geth, self.trace_classifier, block_number, trace_db_session=self.trace_db_session,