From 93c7998e2208355f7bd510f7058b2eb76139cdd4 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Wed, 12 Jan 2022 20:16:31 -0500 Subject: [PATCH 1/4] Allow partial results from the db --- mev_inspect/block.py | 101 +++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 52 deletions(-) diff --git a/mev_inspect/block.py b/mev_inspect/block.py index cf8e2d4..0ff0341 100644 --- a/mev_inspect/block.py +++ b/mev_inspect/block.py @@ -1,6 +1,7 @@ import asyncio import logging -from typing import List, Optional +from collections.abc import Awaitable +from typing import Callable, List, Optional, TypeVar from sqlalchemy import orm from web3 import Web3 @@ -12,6 +13,7 @@ from mev_inspect.schemas.traces import Trace, TraceType from mev_inspect.utils import hex_to_int logger = logging.getLogger(__name__) +T = TypeVar("T") async def get_latest_block_number(base_provider) -> int: @@ -28,62 +30,33 @@ async def create_from_block_number( block_number: int, trace_db_session: Optional[orm.Session], ) -> Block: - block: Optional[Block] = None + existing_block_timestamp = None + existing_base_fee_per_gas = None + existing_traces = None + existing_receipts = None if trace_db_session is not None: - block = _find_block(trace_db_session, block_number) + existing_block_timestamp = _find_block_timestamp(trace_db_session, block_number) + existing_base_fee_per_gas = _find_base_fee(trace_db_session, block_number) + existing_traces = _find_traces(trace_db_session, block_number) + existing_receipts = _find_receipts(trace_db_session, block_number) - if block is None: - block = await _fetch_block(w3, block_number) - return block - else: - return block - - -async def _fetch_block(w3, block_number: int) -> Block: - block_json, receipts_json, traces_json, base_fee_per_gas = await asyncio.gather( - w3.eth.get_block(block_number), - w3.eth.get_block_receipts(block_number), - w3.eth.trace_block(block_number), - fetch_base_fee_per_gas(w3, block_number), + block_timestamp, receipts, traces, base_fee_per_gas = await asyncio.gather( + _await_if_absent( + lambda: _fetch_block_timestamp(w3, block_number), existing_block_timestamp + ), + _await_if_absent( + lambda: _fetch_block_receipts(w3, block_number), existing_receipts + ), + _await_if_absent( + lambda: _fetch_block_traces(w3, block_number), existing_traces + ), + _await_if_absent( + lambda: fetch_base_fee_per_gas(w3, block_number), existing_base_fee_per_gas + ), ) - receipts: List[Receipt] = [Receipt(**receipt) for receipt in receipts_json] - traces = [Trace(**trace_json) for trace_json in traces_json] - - return Block( - block_number=block_number, - block_timestamp=block_json["timestamp"], - miner=block_json["miner"], - base_fee_per_gas=base_fee_per_gas, - traces=traces, - receipts=receipts, - ) - - -def _find_block( - trace_db_session: orm.Session, - block_number: int, -) -> Optional[Block]: - block_timestamp = _find_block_timestamp(trace_db_session, block_number) - if block_timestamp is None: - return None - - base_fee_per_gas = _find_base_fee(trace_db_session, block_number) - if base_fee_per_gas is None: - return None - - traces = _find_traces(trace_db_session, block_number) - if traces is None: - return None - - receipts = _find_receipts(trace_db_session, block_number) - if receipts is None: - return None - miner_address = _get_miner_address_from_traces(traces) - if miner_address is None: - return None return Block( block_number=block_number, @@ -95,6 +68,30 @@ def _find_block( ) +async def _fetch_block_timestamp(w3, block_number: int) -> int: + block_json = await w3.eth.get_block(block_number) + return block_json["timestamp"] + + +async def _fetch_block_receipts(w3, block_number: int) -> List[Receipt]: + receipts_json = await w3.eth.get_block_receipts(block_number) + return [Receipt(**receipt) for receipt in receipts_json] + + +async def _fetch_block_traces(w3, block_number: int) -> List[Trace]: + traces_json = await w3.eth.trace_block(block_number) + return [Trace(**trace_json) for trace_json in traces_json] + + +async def _await_if_absent( + awaitable: Callable[[], Awaitable[T]], existing: Optional[T] +) -> T: + if existing is not None: + return existing + else: + return await awaitable() + + def _find_block_timestamp( trace_db_session: orm.Session, block_number: int, @@ -116,7 +113,7 @@ def _find_traces( block_number: int, ) -> Optional[List[Trace]]: result = trace_db_session.execute( - "SELECT raw_traces FROM block_traces WHERE block_number = :block_number", + "SELECT raw_traces FROM block_traces WHERE block_number = :block_number LIMIT 1", params={"block_number": block_number}, ).one_or_none() From 89d2a718b2eac77bd3515ceee83118351890b8b3 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Tue, 18 Jan 2022 09:24:03 -0500 Subject: [PATCH 2/4] No limits! --- mev_inspect/block.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mev_inspect/block.py b/mev_inspect/block.py index 0ff0341..e58d97e 100644 --- a/mev_inspect/block.py +++ b/mev_inspect/block.py @@ -113,7 +113,7 @@ def _find_traces( block_number: int, ) -> Optional[List[Trace]]: result = trace_db_session.execute( - "SELECT raw_traces FROM block_traces WHERE block_number = :block_number LIMIT 1", + "SELECT raw_traces FROM block_traces WHERE block_number = :block_number", params={"block_number": block_number}, ).one_or_none() From 091ddbd9c197fa9143239376955f5409ed7c294d Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Tue, 18 Jan 2022 11:29:45 -0500 Subject: [PATCH 3/4] Clean up async defaults --- mev_inspect/block.py | 99 +++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 37 deletions(-) diff --git a/mev_inspect/block.py b/mev_inspect/block.py index e58d97e..4c9c876 100644 --- a/mev_inspect/block.py +++ b/mev_inspect/block.py @@ -1,7 +1,6 @@ import asyncio import logging -from collections.abc import Awaitable -from typing import Callable, List, Optional, TypeVar +from typing import List, Optional, TypeVar from sqlalchemy import orm from web3 import Web3 @@ -30,30 +29,11 @@ async def create_from_block_number( block_number: int, trace_db_session: Optional[orm.Session], ) -> Block: - existing_block_timestamp = None - existing_base_fee_per_gas = None - existing_traces = None - existing_receipts = None - - if trace_db_session is not None: - existing_block_timestamp = _find_block_timestamp(trace_db_session, block_number) - existing_base_fee_per_gas = _find_base_fee(trace_db_session, block_number) - existing_traces = _find_traces(trace_db_session, block_number) - existing_receipts = _find_receipts(trace_db_session, block_number) - block_timestamp, receipts, traces, base_fee_per_gas = await asyncio.gather( - _await_if_absent( - lambda: _fetch_block_timestamp(w3, block_number), existing_block_timestamp - ), - _await_if_absent( - lambda: _fetch_block_receipts(w3, block_number), existing_receipts - ), - _await_if_absent( - lambda: _fetch_block_traces(w3, block_number), existing_traces - ), - _await_if_absent( - lambda: fetch_base_fee_per_gas(w3, block_number), existing_base_fee_per_gas - ), + _find_or_fetch_block_timestamp(w3, block_number, trace_db_session), + _find_or_fetch_block_receipts(w3, block_number, trace_db_session), + _find_or_fetch_block_traces(w3, block_number, trace_db_session), + _find_or_fetch_base_fee_per_gas(w3, block_number, trace_db_session), ) miner_address = _get_miner_address_from_traces(traces) @@ -68,6 +48,60 @@ async def create_from_block_number( ) +async def _find_or_fetch_block_timestamp( + w3, + block_number: int, + trace_db_session: Optional[orm.Session], +) -> int: + if trace_db_session is not None: + existing_block_timestamp = _find_block_timestamp(trace_db_session, block_number) + if existing_block_timestamp is not None: + return existing_block_timestamp + + return await _fetch_block_timestamp(w3, block_number) + + +async def _find_or_fetch_block_receipts( + w3, + block_number: int, + trace_db_session: Optional[orm.Session], +) -> List[Receipt]: + if trace_db_session is not None: + existing_block_receipts = _find_block_receipts(trace_db_session, block_number) + if existing_block_receipts is not None: + return existing_block_receipts + + return await _fetch_block_receipts(w3, block_number) + + +async def _find_or_fetch_block_traces( + w3, + block_number: int, + trace_db_session: Optional[orm.Session], +) -> List[Trace]: + if trace_db_session is not None: + existing_block_traces = _find_block_traces(trace_db_session, block_number) + if existing_block_traces is not None: + return existing_block_traces + + return await _fetch_block_traces(w3, block_number) + + +async def _find_or_fetch_base_fee_per_gas( + w3, + block_number: int, + trace_db_session: Optional[orm.Session], +) -> int: + if trace_db_session is not None: + existing_base_fee_per_gas = _find_base_fee_per_gas( + trace_db_session, block_number + ) + if existing_base_fee_per_gas is not None: + return existing_base_fee_per_gas + + return await fetch_base_fee_per_gas(w3, block_number) + + async def _fetch_block_timestamp(w3, block_number: int) -> int: block_json = await w3.eth.get_block(block_number) return block_json["timestamp"] @@ -83,15 +117,6 @@ async def _fetch_block_traces(w3, block_number: int) -> List[Trace]: return [Trace(**trace_json) for trace_json in traces_json] -async def _await_if_absent( - awaitable: Callable[[], Awaitable[T]], existing: Optional[T] -) -> T: - if existing is not None: - return existing - else: - return await awaitable() - - def _find_block_timestamp( trace_db_session: orm.Session, block_number: int, @@ -108,7 +133,7 @@ def _find_block_timestamp( return block_timestamp -def _find_traces( +def _find_block_traces( trace_db_session: orm.Session, block_number: int, ) -> Optional[List[Trace]]: @@ -124,7 +149,7 @@ def _find_traces( return [Trace(**trace_json) for trace_json in traces_json] -def _find_receipts( +def _find_block_receipts( trace_db_session: orm.Session, block_number: int, ) -> Optional[List[Receipt]]: @@ -140,7 +165,7 @@ def _find_receipts( return [Receipt(**receipt) for receipt in receipts_json] -def _find_base_fee( +def _find_base_fee_per_gas( trace_db_session: orm.Session, block_number: int, ) -> Optional[int]: From 85d90e3c6b5cc5269c2b386f5ec4de8f48355397 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Tue, 18 Jan 2022 11:30:32 -0500 Subject: [PATCH 4/4] No need for typevar --- mev_inspect/block.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mev_inspect/block.py b/mev_inspect/block.py index 4c9c876..cf80fa5 100644 --- a/mev_inspect/block.py +++ b/mev_inspect/block.py @@ -1,6 +1,6 @@ import asyncio import logging -from typing import List, Optional, TypeVar +from typing import List, Optional from sqlalchemy import orm from web3 import Web3 @@ -12,7 +12,6 @@ from mev_inspect.schemas.traces import Trace, TraceType from mev_inspect.utils import hex_to_int logger = logging.getLogger(__name__) -T = TypeVar("T") async def get_latest_block_number(base_provider) -> int: