From 2935df284d7cbc2cd6d848f8743bf41be833e3a0 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Sat, 16 Oct 2021 15:48:30 -0400 Subject: [PATCH 1/8] Add optional environment variables for TRACE_DB_* --- k8s/mev-inspect/templates/deployment.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/k8s/mev-inspect/templates/deployment.yaml b/k8s/mev-inspect/templates/deployment.yaml index e06b7dd..9a27d3c 100644 --- a/k8s/mev-inspect/templates/deployment.yaml +++ b/k8s/mev-inspect/templates/deployment.yaml @@ -55,6 +55,24 @@ spec: secretKeyRef: name: mev-inspect-db-credentials key: password + - name: TRACE_DB_HOST + valueFrom: + secretKeyRef: + name: trace-db-credentials + key: host + optional: true + - name: TRACE_DB_USER + valueFrom: + secretKeyRef: + name: trace-db-credentials + key: username + optional: true + - name: TRACE_DB_PASSWORD + valueFrom: + secretKeyRef: + name: trace-db-credentials + key: password + optional: true - name: RPC_URL valueFrom: configMapKeyRef: From 1993f0a14d25955b3c3fea131ae14f93db5a3611 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Sat, 16 Oct 2021 16:28:31 -0400 Subject: [PATCH 2/8] Add trace DB session --- .pre-commit-config.yaml | 10 ++++++---- alembic/env.py | 4 ++-- cli.py | 6 +++--- listener.py | 4 ++-- mev_inspect/db.py | 39 +++++++++++++++++++++++++++++++++------ 5 files changed, 46 insertions(+), 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 397ce75..8ee03eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,11 @@ repos: -- repo: https://github.com/ambv/black - rev: 20.8b1 +- repo: local hooks: - - id: black - language_version: python3.9 + - id: system + name: Black + entry: poetry run black . + pass_filenames: false + language: system - repo: local hooks: - id: pylint diff --git a/alembic/env.py b/alembic/env.py index cbb52c1..7e5e6cd 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -5,12 +5,12 @@ from sqlalchemy import pool from alembic import context -from mev_inspect.db import get_sqlalchemy_database_uri +from mev_inspect.db import get_inspect_database_uri # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config -config.set_main_option("sqlalchemy.url", get_sqlalchemy_database_uri()) +config.set_main_option("sqlalchemy.url", get_inspect_database_uri()) # Interpret the config file for Python logging. # This line sets up loggers basically. diff --git a/cli.py b/cli.py index e7f2386..898fd42 100644 --- a/cli.py +++ b/cli.py @@ -6,7 +6,7 @@ import click from web3 import Web3 from mev_inspect.classifiers.trace import TraceClassifier -from mev_inspect.db import get_session +from mev_inspect.db import get_inspect_session from mev_inspect.inspect_block import inspect_block from mev_inspect.provider import get_base_provider @@ -27,7 +27,7 @@ def cli(): @click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, "")) @click.option("--cache/--no-cache", default=True) def inspect_block_command(block_number: int, rpc: str, cache: bool): - db_session = get_session() + db_session = get_inspect_session() base_provider = get_base_provider(rpc) w3 = Web3(base_provider) trace_classifier = TraceClassifier() @@ -54,7 +54,7 @@ def inspect_many_blocks_command( after_block: int, before_block: int, rpc: str, cache: bool ): - db_session = get_session() + db_session = get_inspect_session() base_provider = get_base_provider(rpc) w3 = Web3(base_provider) trace_classifier = TraceClassifier() diff --git a/listener.py b/listener.py index 88e364c..05cfb38 100644 --- a/listener.py +++ b/listener.py @@ -9,7 +9,7 @@ from mev_inspect.crud.latest_block_update import ( find_latest_block_update, update_latest_block, ) -from mev_inspect.db import get_session +from mev_inspect.db import get_inspect_session from mev_inspect.inspect_block import inspect_block from mev_inspect.provider import get_base_provider from mev_inspect.signal_handler import GracefulKiller @@ -32,7 +32,7 @@ def run(): killer = GracefulKiller() - db_session = get_session() + db_session = get_inspect_session() base_provider = get_base_provider(rpc) w3 = Web3(base_provider) diff --git a/mev_inspect/db.py b/mev_inspect/db.py index df2b7d9..9cdaa48 100644 --- a/mev_inspect/db.py +++ b/mev_inspect/db.py @@ -1,10 +1,23 @@ import os +from typing import Optional -from sqlalchemy import create_engine +from sqlalchemy import create_engine, orm from sqlalchemy.orm import sessionmaker -def get_sqlalchemy_database_uri(): +def get_trace_database_uri() -> Optional[str]: + username = os.getenv("TRACE_DB_USER") + password = os.getenv("TRACE_DB_PASSWORD") + host = os.getenv("TRACE_DB_HOST") + db_name = "trace_db" + + if all(field is not None for field in [username, password, host]): + return f"postgresql://{username}:{password}@{host}/{db_name}" + + return None + + +def get_inspect_database_uri(): username = os.getenv("POSTGRES_USER") password = os.getenv("POSTGRES_PASSWORD") host = os.getenv("POSTGRES_HOST") @@ -12,10 +25,24 @@ def get_sqlalchemy_database_uri(): return f"postgresql://{username}:{password}@{host}/{db_name}" -def get_engine(): - return create_engine(get_sqlalchemy_database_uri()) +def _get_engine(uri: str): + return create_engine(uri) -def get_session(): - Session = sessionmaker(bind=get_engine()) +def _get_session(uri: str): + Session = sessionmaker(bind=_get_engine(uri)) return Session() + + +def get_inspect_session() -> orm.Session: + uri = get_inspect_database_uri() + return _get_session(uri) + + +def get_trace_session() -> Optional[orm.Session]: + uri = get_trace_database_uri() + + if uri is not None: + return _get_session(uri) + + return None From ee5a4905e664a236dd9278ef9957811a32ff1513 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Sun, 17 Oct 2021 02:13:44 -0400 Subject: [PATCH 3/8] Weave trace db through to fetch --- .pre-commit-config.yaml | 2 +- cli.py | 16 ++++++--- mev_inspect/block.py | 21 ++++++++---- mev_inspect/inspect_block.py | 30 +++++++++-------- poetry.lock | 64 +----------------------------------- pyproject.toml | 1 - 6 files changed, 45 insertions(+), 89 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ee03eb..50726ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ repos: - repo: local hooks: - id: system - name: Black + name: black entry: poetry run black . pass_filenames: false language: system diff --git a/cli.py b/cli.py index 898fd42..5eeaa29 100644 --- a/cli.py +++ b/cli.py @@ -6,7 +6,7 @@ import click from web3 import Web3 from mev_inspect.classifiers.trace import TraceClassifier -from mev_inspect.db import get_inspect_session +from mev_inspect.db import get_inspect_session, get_trace_session from mev_inspect.inspect_block import inspect_block from mev_inspect.provider import get_base_provider @@ -27,7 +27,9 @@ def cli(): @click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, "")) @click.option("--cache/--no-cache", default=True) def inspect_block_command(block_number: int, rpc: str, cache: bool): - db_session = get_inspect_session() + inspect_db_session = get_inspect_session() + trace_db_session = get_trace_session() + base_provider = get_base_provider(rpc) w3 = Web3(base_provider) trace_classifier = TraceClassifier() @@ -36,12 +38,13 @@ def inspect_block_command(block_number: int, rpc: str, cache: bool): logger.info("Skipping cache") inspect_block( - db_session, + inspect_db_session, base_provider, w3, trace_classifier, block_number, should_cache=cache, + trace_db_session=trace_db_session, ) @@ -54,7 +57,9 @@ def inspect_many_blocks_command( after_block: int, before_block: int, rpc: str, cache: bool ): - db_session = get_inspect_session() + inspect_db_session = get_inspect_session() + trace_db_session = get_trace_session() + base_provider = get_base_provider(rpc) w3 = Web3(base_provider) trace_classifier = TraceClassifier() @@ -72,13 +77,14 @@ def inspect_many_blocks_command( logger.info(dashes) inspect_block( - db_session, + inspect_db_session, base_provider, w3, trace_classifier, block_number, should_write_classified_traces=False, should_cache=cache, + trace_db_session=trace_db_session, ) diff --git a/mev_inspect/block.py b/mev_inspect/block.py index 1aa874b..9dd7c48 100644 --- a/mev_inspect/block.py +++ b/mev_inspect/block.py @@ -1,6 +1,7 @@ from pathlib import Path -from typing import List +from typing import List, Optional +from sqlalchemy import orm from web3 import Web3 from mev_inspect.fees import fetch_base_fee_per_gas @@ -16,28 +17,36 @@ def get_latest_block_number(w3: Web3) -> int: def create_from_block_number( - base_provider, w3: Web3, block_number: int, should_cache: bool + base_provider, + w3: Web3, + block_number: int, + should_cache: bool, + trace_db_session: Optional[orm.Session], ) -> Block: if not should_cache: - return fetch_block(w3, base_provider, block_number) + return fetch_block(w3, base_provider, block_number, trace_db_session) cache_path = _get_cache_path(block_number) if cache_path.is_file(): print(f"Cache for block {block_number} exists, " "loading data from cache") - return Block.parse_file(cache_path) else: print(f"Cache for block {block_number} did not exist, getting data") - block = fetch_block(w3, base_provider, block_number) + block = fetch_block(w3, base_provider, block_number, trace_db_session) cache_block(cache_path, block) return block -def fetch_block(w3, base_provider, block_number: int) -> Block: +def fetch_block( + w3, + base_provider, + block_number: int, + trace_db_session: Optional[orm.Session], # pylint: disable=unused-argument +) -> Block: block_json = w3.eth.get_block(block_number) receipts_json = base_provider.make_request("eth_getBlockReceipts", [block_number]) traces_json = w3.parity.trace_block(block_number) diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py index a2e91e4..d728932 100644 --- a/mev_inspect/inspect_block.py +++ b/mev_inspect/inspect_block.py @@ -1,5 +1,7 @@ import logging +from typing import Optional +from sqlalchemy import orm from web3 import Web3 from mev_inspect.arbitrages import get_arbitrages @@ -34,12 +36,13 @@ logger = logging.getLogger(__name__) def inspect_block( - db_session, + inspect_db_session: orm.Session, base_provider, w3: Web3, trace_clasifier: TraceClassifier, block_number: int, should_cache: bool, + trace_db_session: Optional[orm.Session], should_write_classified_traces: bool = True, ): block = create_from_block_number( @@ -47,6 +50,7 @@ def inspect_block( w3, block_number, should_cache, + trace_db_session, ) logger.info(f"Total traces: {len(block.traces)}") @@ -60,36 +64,36 @@ def inspect_block( logger.info(f"Returned {len(classified_traces)} classified traces") if should_write_classified_traces: - delete_classified_traces_for_block(db_session, block_number) - write_classified_traces(db_session, classified_traces) + delete_classified_traces_for_block(inspect_db_session, block_number) + write_classified_traces(inspect_db_session, classified_traces) transfers = get_transfers(classified_traces) logger.info(f"Found {len(transfers)} transfers") - delete_transfers_for_block(db_session, block_number) - write_transfers(db_session, transfers) + delete_transfers_for_block(inspect_db_session, block_number) + write_transfers(inspect_db_session, transfers) swaps = get_swaps(classified_traces) logger.info(f"Found {len(swaps)} swaps") - delete_swaps_for_block(db_session, block_number) - write_swaps(db_session, swaps) + delete_swaps_for_block(inspect_db_session, block_number) + write_swaps(inspect_db_session, swaps) arbitrages = get_arbitrages(swaps) logger.info(f"Found {len(arbitrages)} arbitrages") - delete_arbitrages_for_block(db_session, block_number) - write_arbitrages(db_session, arbitrages) + delete_arbitrages_for_block(inspect_db_session, block_number) + write_arbitrages(inspect_db_session, arbitrages) liquidations = get_liquidations(classified_traces) logger.info(f"Found {len(liquidations)} liquidations") - delete_liquidations_for_block(db_session, block_number) - write_liquidations(db_session, liquidations) + delete_liquidations_for_block(inspect_db_session, block_number) + write_liquidations(inspect_db_session, liquidations) miner_payments = get_miner_payments( block.miner, block.base_fee_per_gas, classified_traces, block.receipts ) - delete_miner_payments_for_block(db_session, block_number) - write_miner_payments(db_session, miner_payments) + delete_miner_payments_for_block(inspect_db_session, block_number) + write_miner_payments(inspect_db_session, miner_payments) diff --git a/poetry.lock b/poetry.lock index 586ae29..ccd2565 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,14 +31,6 @@ python-dateutil = "*" python-editor = ">=0.3" SQLAlchemy = ">=1.3.0" -[[package]] -name = "appdirs" -version = "1.4.4" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "astroid" version = "2.7.2" @@ -112,28 +104,6 @@ category = "main" optional = false python-versions = "*" -[[package]] -name = "black" -version = "21.7b0" -description = "The uncompromising code formatter." -category = "dev" -optional = false -python-versions = ">=3.6.2" - -[package.dependencies] -appdirs = "*" -click = ">=7.1.2" -mypy-extensions = ">=0.4.3" -pathspec = ">=0.8.1,<1" -regex = ">=2020.1.8" -tomli = ">=0.2.6,<2.0.0" - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.6.0)", "aiohttp-cors (>=0.4.0)"] -python2 = ["typed-ast (>=1.4.2)"] -uvloop = ["uvloop (>=0.15.2)"] - [[package]] name = "bottle" version = "0.12.19" @@ -625,14 +595,6 @@ python-versions = "*" [package.dependencies] six = ">=1.9.0" -[[package]] -name = "pathspec" -version = "0.9.0" -description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - [[package]] name = "platformdirs" version = "2.2.0" @@ -938,14 +900,6 @@ category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -[[package]] -name = "tomli" -version = "1.2.1" -description = "A lil' TOML parser" -category = "dev" -optional = false -python-versions = ">=3.6" - [[package]] name = "toolz" version = "0.11.1" @@ -1063,7 +1017,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "7b8511360be2f67f85c4e9e2d5ee73da2ce14cc691a44e8c6cb03552af6d75b5" +content-hash = "baade6f62f3adaff192b2c85b4f602f4990b9b99d6fcce904aeb5087b6fa1921" [metadata.files] aiohttp = [ @@ -1109,10 +1063,6 @@ alembic = [ {file = "alembic-1.6.5-py2.py3-none-any.whl", hash = "sha256:e78be5b919f5bb184e3e0e2dd1ca986f2362e29a2bc933c446fe89f39dbe4e9c"}, {file = "alembic-1.6.5.tar.gz", hash = "sha256:a21fedebb3fb8f6bbbba51a11114f08c78709377051384c9c5ead5705ee93a51"}, ] -appdirs = [ - {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, - {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, -] astroid = [ {file = "astroid-2.7.2-py3-none-any.whl", hash = "sha256:ecc50f9b3803ebf8ea19aa2c6df5622d8a5c31456a53c741d3be044d96ff0948"}, {file = "astroid-2.7.2.tar.gz", hash = "sha256:b6c2d75cd7c2982d09e7d41d70213e863b3ba34d3bd4014e08f167cee966e99e"}, @@ -1140,10 +1090,6 @@ base58 = [ bitarray = [ {file = "bitarray-1.2.2.tar.gz", hash = "sha256:27a69ffcee3b868abab3ce8b17c69e02b63e722d4d64ffd91d659f81e9984954"}, ] -black = [ - {file = "black-21.7b0-py3-none-any.whl", hash = "sha256:1c7aa6ada8ee864db745b22790a32f94b2795c253a75d6d9b5e439ff10d23116"}, - {file = "black-21.7b0.tar.gz", hash = "sha256:c8373c6491de9362e39271630b65b964607bc5c79c83783547d76c839b3aa219"}, -] bottle = [ {file = "bottle-0.12.19-py3-none-any.whl", hash = "sha256:f6b8a34fe9aa406f9813c02990db72ca69ce6a158b5b156d2c41f345016a723d"}, {file = "bottle-0.12.19.tar.gz", hash = "sha256:a9d73ffcbc6a1345ca2d7949638db46349f5b2b77dac65d6494d45c23628da2c"}, @@ -1550,10 +1496,6 @@ packaging = [ parsimonious = [ {file = "parsimonious-0.8.1.tar.gz", hash = "sha256:3add338892d580e0cb3b1a39e4a1b427ff9f687858fdd61097053742391a9f6b"}, ] -pathspec = [ - {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, - {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, -] platformdirs = [ {file = "platformdirs-2.2.0-py3-none-any.whl", hash = "sha256:4666d822218db6a262bdfdc9c39d21f23b4cfdb08af331a81e92751daf6c866c"}, {file = "platformdirs-2.2.0.tar.gz", hash = "sha256:632daad3ab546bd8e6af0537d09805cec458dce201bccfe23012df73332e181e"}, @@ -1862,10 +1804,6 @@ toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] -tomli = [ - {file = "tomli-1.2.1-py3-none-any.whl", hash = "sha256:8dd0e9524d6f386271a36b41dbf6c57d8e32fd96fd22b6584679dc569d20899f"}, - {file = "tomli-1.2.1.tar.gz", hash = "sha256:a5b75cb6f3968abb47af1b40c1819dc519ea82bcc065776a866e8d74c5ca9442"}, -] toolz = [ {file = "toolz-0.11.1-py3-none-any.whl", hash = "sha256:1bc473acbf1a1db4e72a1ce587be347450e8f08324908b8a266b486f408f04d5"}, {file = "toolz-0.11.1.tar.gz", hash = "sha256:c7a47921f07822fe534fb1c01c9931ab335a4390c782bd28c6bcc7c2f71f3fbf"}, diff --git a/pyproject.toml b/pyproject.toml index e4275be..797a99c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ pytest-sugar = "^0.9.4" pytest-cov = "^2.12.1" coverage = "^5.5" alembic = "^1.6.5" -black = "^21.7b0" CProfileV = "^1.0.7" regex = "^2021.10.8" From dbc7c5d4aeda33a0e4358011018ce912f32d8fd4 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Sun, 17 Oct 2021 02:44:39 -0400 Subject: [PATCH 4/8] Add credentials to tiltfile. Find traces using traces db --- Tiltfile | 7 +++++++ mev_inspect/block.py | 27 +++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/Tiltfile b/Tiltfile index b358834..6675eca 100644 --- a/Tiltfile +++ b/Tiltfile @@ -19,6 +19,13 @@ k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = { "host": "postgresql", })) +# if using https://github.com/taarushv/trace-db +# k8s_yaml(secret_from_dict("trace-db-credentials", inputs = { +# "username" : "username", +# "password": "password", +# "host": "trace-db-postgresql", +# })) + docker_build_with_restart("mev-inspect-py", ".", entrypoint="/app/entrypoint.sh", live_update=[ diff --git a/mev_inspect/block.py b/mev_inspect/block.py index 9dd7c48..d732d58 100644 --- a/mev_inspect/block.py +++ b/mev_inspect/block.py @@ -45,11 +45,18 @@ def fetch_block( w3, base_provider, block_number: int, - trace_db_session: Optional[orm.Session], # pylint: disable=unused-argument + trace_db_session: Optional[orm.Session], ) -> Block: block_json = w3.eth.get_block(block_number) receipts_json = base_provider.make_request("eth_getBlockReceipts", [block_number]) - traces_json = w3.parity.trace_block(block_number) + + traces_json: Optional[List[dict]] = None + + if trace_db_session is not None: + traces_json = find_traces(trace_db_session, block_number) + + if traces_json is None: + traces_json = w3.parity.trace_block(block_number) receipts: List[Receipt] = [ Receipt(**receipt) for receipt in receipts_json["result"] @@ -66,6 +73,22 @@ def fetch_block( ) +def find_traces( + trace_db_session: orm.Session, + block_number: int, +) -> Optional[List[dict]]: + result = trace_db_session.execute( + "SELECT raw_traces FROM block_traces WHERE block_number = :block_number", + params={"block_number": block_number}, + ).one_or_none() + + if result is None: + return None + else: + (raw_traces,) = result + return raw_traces + + def get_transaction_hashes(calls: List[Trace]) -> List[str]: result = [] From a786b74f4ab791452ba3994abc94a9ebcdac7ef9 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Sun, 17 Oct 2021 11:08:51 -0400 Subject: [PATCH 5/8] Update listener --- listener.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/listener.py b/listener.py index 05cfb38..d68cfd8 100644 --- a/listener.py +++ b/listener.py @@ -9,7 +9,8 @@ from mev_inspect.crud.latest_block_update import ( find_latest_block_update, update_latest_block, ) -from mev_inspect.db import get_inspect_session +from mev_inspect.classifiers.trace import TraceClassifier +from mev_inspect.db import get_inspect_session, get_trace_session from mev_inspect.inspect_block import inspect_block from mev_inspect.provider import get_base_provider from mev_inspect.signal_handler import GracefulKiller @@ -18,7 +19,6 @@ from mev_inspect.signal_handler import GracefulKiller logging.basicConfig(filename="listener.log", level=logging.INFO) logger = logging.getLogger(__name__) - # lag to make sure the blocks we see are settled BLOCK_NUMBER_LAG = 5 @@ -32,14 +32,17 @@ def run(): killer = GracefulKiller() - db_session = get_inspect_session() + inspect_db_session = get_inspect_session() + trace_db_session = get_trace_session() + trace_classifier = TraceClassifier() + base_provider = get_base_provider(rpc) w3 = Web3(base_provider) latest_block_number = get_latest_block_number(w3) while not killer.kill_now: - last_written_block = find_latest_block_update(db_session) + last_written_block = find_latest_block_update(inspect_db_session) logger.info(f"Latest block: {latest_block_number}") logger.info(f"Last written block: {last_written_block}") @@ -55,14 +58,16 @@ def run(): logger.info(f"Writing block: {block_number}") inspect_block( - db_session, + inspect_db_session, base_provider, w3, + trace_classifier, block_number, should_write_classified_traces=False, should_cache=False, + trace_db_session=trace_db_session, ) - update_latest_block(db_session, block_number) + update_latest_block(inspect_db_session, block_number) else: time.sleep(5) latest_block_number = get_latest_block_number(w3) @@ -71,4 +76,7 @@ def run(): if __name__ == "__main__": - run() + try: + run() + except Exception as e: + logger.error(e) From 68232f4161373fce454b03b17ddd850d4895f031 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 18 Oct 2021 11:33:14 -0400 Subject: [PATCH 6/8] Fetch all of the block from the DB if possible --- cli.py | 2 - listener.py | 1 - mev_inspect/block.py | 103 ++++++++++++++++++++++++++--------- mev_inspect/inspect_block.py | 2 - 4 files changed, 76 insertions(+), 32 deletions(-) diff --git a/cli.py b/cli.py index 5eeaa29..506dea0 100644 --- a/cli.py +++ b/cli.py @@ -43,7 +43,6 @@ def inspect_block_command(block_number: int, rpc: str, cache: bool): w3, trace_classifier, block_number, - should_cache=cache, trace_db_session=trace_db_session, ) @@ -83,7 +82,6 @@ def inspect_many_blocks_command( trace_classifier, block_number, should_write_classified_traces=False, - should_cache=cache, trace_db_session=trace_db_session, ) diff --git a/listener.py b/listener.py index d68cfd8..928603f 100644 --- a/listener.py +++ b/listener.py @@ -64,7 +64,6 @@ def run(): trace_classifier, block_number, should_write_classified_traces=False, - should_cache=False, trace_db_session=trace_db_session, ) update_latest_block(inspect_db_session, block_number) diff --git a/mev_inspect/block.py b/mev_inspect/block.py index d732d58..ab4afe4 100644 --- a/mev_inspect/block.py +++ b/mev_inspect/block.py @@ -20,43 +20,27 @@ def create_from_block_number( base_provider, w3: Web3, block_number: int, - should_cache: bool, trace_db_session: Optional[orm.Session], ) -> Block: - if not should_cache: - return fetch_block(w3, base_provider, block_number, trace_db_session) + block: Optional[Block] = None - cache_path = _get_cache_path(block_number) + if trace_db_session is not None: + block = _find_block(trace_db_session, block_number) - if cache_path.is_file(): - print(f"Cache for block {block_number} exists, " "loading data from cache") - return Block.parse_file(cache_path) + if block is None: + return _fetch_block(w3, base_provider, block_number) else: - print(f"Cache for block {block_number} did not exist, getting data") - - block = fetch_block(w3, base_provider, block_number, trace_db_session) - - cache_block(cache_path, block) - return block -def fetch_block( +def _fetch_block( w3, base_provider, block_number: int, - trace_db_session: Optional[orm.Session], ) -> Block: block_json = w3.eth.get_block(block_number) receipts_json = base_provider.make_request("eth_getBlockReceipts", [block_number]) - - traces_json: Optional[List[dict]] = None - - if trace_db_session is not None: - traces_json = find_traces(trace_db_session, block_number) - - if traces_json is None: - traces_json = w3.parity.trace_block(block_number) + traces_json = w3.parity.trace_block(block_number) receipts: List[Receipt] = [ Receipt(**receipt) for receipt in receipts_json["result"] @@ -73,10 +57,35 @@ def fetch_block( ) -def find_traces( +def _find_block( trace_db_session: orm.Session, block_number: int, -) -> Optional[List[dict]]: +) -> Optional[Block]: + traces = _find_traces(trace_db_session, block_number) + receipts = _find_receipts(trace_db_session, block_number) + base_fee_per_gas = _find_base_fee(trace_db_session, block_number) + + if traces is None or receipts is None or base_fee_per_gas is None: + return None + + miner_address = _get_miner_address_from_traces(traces) + + if miner_address is None: + return None + + return Block( + block_number=block_number, + miner=miner_address, + base_fee_per_gas=base_fee_per_gas, + traces=traces, + receipts=receipts, + ) + + +def _find_traces( + trace_db_session: orm.Session, + block_number: int, +) -> Optional[List[Trace]]: result = trace_db_session.execute( "SELECT raw_traces FROM block_traces WHERE block_number = :block_number", params={"block_number": block_number}, @@ -85,8 +94,48 @@ def find_traces( if result is None: return None else: - (raw_traces,) = result - return raw_traces + (traces_json,) = result + return [Trace(**trace_json) for trace_json in traces_json] + + +def _find_receipts( + trace_db_session: orm.Session, + block_number: int, +) -> Optional[List[Receipt]]: + result = trace_db_session.execute( + "SELECT raw_receipts FROM block_receipts WHERE block_number = :block_number", + params={"block_number": block_number}, + ).one_or_none() + + if result is None: + return None + else: + (receipts_json,) = result + return [Receipt(**receipt) for receipt in receipts_json] + + +def _find_base_fee( + trace_db_session: orm.Session, + block_number: int, +) -> Optional[int]: + result = trace_db_session.execute( + "SELECT base_fee_in_wei FROM base_fee WHERE block_number = :block_number", + params={"block_number": block_number}, + ).one_or_none() + + if result is None: + return None + else: + (base_fee,) = result + return base_fee + + +def _get_miner_address_from_traces(traces: List[Trace]) -> Optional[str]: + for trace in traces: + if trace.type == TraceType.reward: + return trace.action["author"] + + return None def get_transaction_hashes(calls: List[Trace]) -> List[str]: diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py index d728932..d3ab75b 100644 --- a/mev_inspect/inspect_block.py +++ b/mev_inspect/inspect_block.py @@ -41,7 +41,6 @@ def inspect_block( w3: Web3, trace_clasifier: TraceClassifier, block_number: int, - should_cache: bool, trace_db_session: Optional[orm.Session], should_write_classified_traces: bool = True, ): @@ -49,7 +48,6 @@ def inspect_block( base_provider, w3, block_number, - should_cache, trace_db_session, ) From 063b8764a859d5889d5ddbef32f449adc7b2201e Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 18 Oct 2021 12:01:26 -0400 Subject: [PATCH 7/8] Write classified traces --- cli.py | 1 - listener.py | 1 - 2 files changed, 2 deletions(-) diff --git a/cli.py b/cli.py index 506dea0..fb8a080 100644 --- a/cli.py +++ b/cli.py @@ -81,7 +81,6 @@ def inspect_many_blocks_command( w3, trace_classifier, block_number, - should_write_classified_traces=False, trace_db_session=trace_db_session, ) diff --git a/listener.py b/listener.py index 928603f..5c1e386 100644 --- a/listener.py +++ b/listener.py @@ -63,7 +63,6 @@ def run(): w3, trace_classifier, block_number, - should_write_classified_traces=False, trace_db_session=trace_db_session, ) update_latest_block(inspect_db_session, block_number) From 4ae59b8e281e85b102e49e1975564714d9498d10 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 18 Oct 2021 12:58:57 -0400 Subject: [PATCH 8/8] Back to original --- .pre-commit-config.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 50726ce..397ce75 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,9 @@ repos: -- repo: local +- repo: https://github.com/ambv/black + rev: 20.8b1 hooks: - - id: system - name: black - entry: poetry run black . - pass_filenames: false - language: system + - id: black + language_version: python3.9 - repo: local hooks: - id: pylint