From ee5a4905e664a236dd9278ef9957811a32ff1513 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Sun, 17 Oct 2021 02:13:44 -0400 Subject: [PATCH] Weave trace db through to fetch --- .pre-commit-config.yaml | 2 +- cli.py | 16 ++++++--- mev_inspect/block.py | 21 ++++++++---- mev_inspect/inspect_block.py | 30 +++++++++-------- poetry.lock | 64 +----------------------------------- pyproject.toml | 1 - 6 files changed, 45 insertions(+), 89 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ee03eb..50726ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ repos: - repo: local hooks: - id: system - name: Black + name: black entry: poetry run black . pass_filenames: false language: system diff --git a/cli.py b/cli.py index 898fd42..5eeaa29 100644 --- a/cli.py +++ b/cli.py @@ -6,7 +6,7 @@ import click from web3 import Web3 from mev_inspect.classifiers.trace import TraceClassifier -from mev_inspect.db import get_inspect_session +from mev_inspect.db import get_inspect_session, get_trace_session from mev_inspect.inspect_block import inspect_block from mev_inspect.provider import get_base_provider @@ -27,7 +27,9 @@ def cli(): @click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, "")) @click.option("--cache/--no-cache", default=True) def inspect_block_command(block_number: int, rpc: str, cache: bool): - db_session = get_inspect_session() + inspect_db_session = get_inspect_session() + trace_db_session = get_trace_session() + base_provider = get_base_provider(rpc) w3 = Web3(base_provider) trace_classifier = TraceClassifier() @@ -36,12 +38,13 @@ def inspect_block_command(block_number: int, rpc: str, cache: bool): logger.info("Skipping cache") inspect_block( - db_session, + inspect_db_session, base_provider, w3, trace_classifier, block_number, should_cache=cache, + trace_db_session=trace_db_session, ) @@ -54,7 +57,9 @@ def inspect_many_blocks_command( after_block: int, before_block: int, rpc: str, cache: bool ): - db_session = get_inspect_session() + inspect_db_session = get_inspect_session() + trace_db_session = get_trace_session() + base_provider = get_base_provider(rpc) w3 = Web3(base_provider) trace_classifier = TraceClassifier() @@ -72,13 +77,14 @@ def inspect_many_blocks_command( logger.info(dashes) inspect_block( - db_session, + inspect_db_session, base_provider, w3, trace_classifier, block_number, should_write_classified_traces=False, should_cache=cache, + trace_db_session=trace_db_session, ) diff --git a/mev_inspect/block.py b/mev_inspect/block.py index 1aa874b..9dd7c48 100644 --- a/mev_inspect/block.py +++ b/mev_inspect/block.py @@ -1,6 +1,7 @@ from pathlib import Path -from typing import List +from typing import List, Optional +from sqlalchemy import orm from web3 import Web3 from mev_inspect.fees import fetch_base_fee_per_gas @@ -16,28 +17,36 @@ def get_latest_block_number(w3: Web3) -> int: def create_from_block_number( - base_provider, w3: Web3, block_number: int, should_cache: bool + base_provider, + w3: Web3, + block_number: int, + should_cache: bool, + trace_db_session: Optional[orm.Session], ) -> Block: if not should_cache: - return fetch_block(w3, base_provider, block_number) + return fetch_block(w3, base_provider, block_number, trace_db_session) cache_path = _get_cache_path(block_number) if cache_path.is_file(): print(f"Cache for block {block_number} exists, " "loading data from cache") - return Block.parse_file(cache_path) else: print(f"Cache for block {block_number} did not exist, getting data") - block = fetch_block(w3, base_provider, block_number) + block = fetch_block(w3, base_provider, block_number, trace_db_session) cache_block(cache_path, block) return block -def fetch_block(w3, base_provider, block_number: int) -> Block: +def fetch_block( + w3, + base_provider, + block_number: int, + trace_db_session: Optional[orm.Session], # pylint: disable=unused-argument +) -> Block: block_json = w3.eth.get_block(block_number) receipts_json = base_provider.make_request("eth_getBlockReceipts", [block_number]) traces_json = w3.parity.trace_block(block_number) diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py index a2e91e4..d728932 100644 --- a/mev_inspect/inspect_block.py +++ b/mev_inspect/inspect_block.py @@ -1,5 +1,7 @@ import logging +from typing import Optional +from sqlalchemy import orm from web3 import Web3 from mev_inspect.arbitrages import get_arbitrages @@ -34,12 +36,13 @@ logger = logging.getLogger(__name__) def inspect_block( - db_session, + inspect_db_session: orm.Session, base_provider, w3: Web3, trace_clasifier: TraceClassifier, block_number: int, should_cache: bool, + trace_db_session: Optional[orm.Session], should_write_classified_traces: bool = True, ): block = create_from_block_number( @@ -47,6 +50,7 @@ def inspect_block( w3, block_number, should_cache, + trace_db_session, ) logger.info(f"Total traces: {len(block.traces)}") @@ -60,36 +64,36 @@ def inspect_block( logger.info(f"Returned {len(classified_traces)} classified traces") if should_write_classified_traces: - delete_classified_traces_for_block(db_session, block_number) - write_classified_traces(db_session, classified_traces) + delete_classified_traces_for_block(inspect_db_session, block_number) + write_classified_traces(inspect_db_session, classified_traces) transfers = get_transfers(classified_traces) logger.info(f"Found {len(transfers)} transfers") - delete_transfers_for_block(db_session, block_number) - write_transfers(db_session, transfers) + delete_transfers_for_block(inspect_db_session, block_number) + write_transfers(inspect_db_session, transfers) swaps = get_swaps(classified_traces) logger.info(f"Found {len(swaps)} swaps") - delete_swaps_for_block(db_session, block_number) - write_swaps(db_session, swaps) + delete_swaps_for_block(inspect_db_session, block_number) + write_swaps(inspect_db_session, swaps) arbitrages = get_arbitrages(swaps) logger.info(f"Found {len(arbitrages)} arbitrages") - delete_arbitrages_for_block(db_session, block_number) - write_arbitrages(db_session, arbitrages) + delete_arbitrages_for_block(inspect_db_session, block_number) + write_arbitrages(inspect_db_session, arbitrages) liquidations = get_liquidations(classified_traces) logger.info(f"Found {len(liquidations)} liquidations") - delete_liquidations_for_block(db_session, block_number) - write_liquidations(db_session, liquidations) + delete_liquidations_for_block(inspect_db_session, block_number) + write_liquidations(inspect_db_session, liquidations) miner_payments = get_miner_payments( block.miner, block.base_fee_per_gas, classified_traces, block.receipts ) - delete_miner_payments_for_block(db_session, block_number) - write_miner_payments(db_session, miner_payments) + delete_miner_payments_for_block(inspect_db_session, block_number) + write_miner_payments(inspect_db_session, miner_payments) diff --git a/poetry.lock b/poetry.lock index 586ae29..ccd2565 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,14 +31,6 @@ python-dateutil = "*" python-editor = ">=0.3" SQLAlchemy = ">=1.3.0" -[[package]] -name = "appdirs" -version = "1.4.4" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "astroid" version = "2.7.2" @@ -112,28 +104,6 @@ category = "main" optional = false python-versions = "*" -[[package]] -name = "black" -version = "21.7b0" -description = "The uncompromising code formatter." -category = "dev" -optional = false -python-versions = ">=3.6.2" - -[package.dependencies] -appdirs = "*" -click = ">=7.1.2" -mypy-extensions = ">=0.4.3" -pathspec = ">=0.8.1,<1" -regex = ">=2020.1.8" -tomli = ">=0.2.6,<2.0.0" - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.6.0)", "aiohttp-cors (>=0.4.0)"] -python2 = ["typed-ast (>=1.4.2)"] -uvloop = ["uvloop (>=0.15.2)"] - [[package]] name = "bottle" version = "0.12.19" @@ -625,14 +595,6 @@ python-versions = "*" [package.dependencies] six = ">=1.9.0" -[[package]] -name = "pathspec" -version = "0.9.0" -description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - [[package]] name = "platformdirs" version = "2.2.0" @@ -938,14 +900,6 @@ category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -[[package]] -name = "tomli" -version = "1.2.1" -description = "A lil' TOML parser" -category = "dev" -optional = false -python-versions = ">=3.6" - [[package]] name = "toolz" version = "0.11.1" @@ -1063,7 +1017,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "7b8511360be2f67f85c4e9e2d5ee73da2ce14cc691a44e8c6cb03552af6d75b5" +content-hash = "baade6f62f3adaff192b2c85b4f602f4990b9b99d6fcce904aeb5087b6fa1921" [metadata.files] aiohttp = [ @@ -1109,10 +1063,6 @@ alembic = [ {file = "alembic-1.6.5-py2.py3-none-any.whl", hash = "sha256:e78be5b919f5bb184e3e0e2dd1ca986f2362e29a2bc933c446fe89f39dbe4e9c"}, {file = "alembic-1.6.5.tar.gz", hash = "sha256:a21fedebb3fb8f6bbbba51a11114f08c78709377051384c9c5ead5705ee93a51"}, ] -appdirs = [ - {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, - {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, -] astroid = [ {file = "astroid-2.7.2-py3-none-any.whl", hash = "sha256:ecc50f9b3803ebf8ea19aa2c6df5622d8a5c31456a53c741d3be044d96ff0948"}, {file = "astroid-2.7.2.tar.gz", hash = "sha256:b6c2d75cd7c2982d09e7d41d70213e863b3ba34d3bd4014e08f167cee966e99e"}, @@ -1140,10 +1090,6 @@ base58 = [ bitarray = [ {file = "bitarray-1.2.2.tar.gz", hash = "sha256:27a69ffcee3b868abab3ce8b17c69e02b63e722d4d64ffd91d659f81e9984954"}, ] -black = [ - {file = "black-21.7b0-py3-none-any.whl", hash = "sha256:1c7aa6ada8ee864db745b22790a32f94b2795c253a75d6d9b5e439ff10d23116"}, - {file = "black-21.7b0.tar.gz", hash = "sha256:c8373c6491de9362e39271630b65b964607bc5c79c83783547d76c839b3aa219"}, -] bottle = [ {file = "bottle-0.12.19-py3-none-any.whl", hash = "sha256:f6b8a34fe9aa406f9813c02990db72ca69ce6a158b5b156d2c41f345016a723d"}, {file = "bottle-0.12.19.tar.gz", hash = "sha256:a9d73ffcbc6a1345ca2d7949638db46349f5b2b77dac65d6494d45c23628da2c"}, @@ -1550,10 +1496,6 @@ packaging = [ parsimonious = [ {file = "parsimonious-0.8.1.tar.gz", hash = "sha256:3add338892d580e0cb3b1a39e4a1b427ff9f687858fdd61097053742391a9f6b"}, ] -pathspec = [ - {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, - {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, -] platformdirs = [ {file = "platformdirs-2.2.0-py3-none-any.whl", hash = "sha256:4666d822218db6a262bdfdc9c39d21f23b4cfdb08af331a81e92751daf6c866c"}, {file = "platformdirs-2.2.0.tar.gz", hash = "sha256:632daad3ab546bd8e6af0537d09805cec458dce201bccfe23012df73332e181e"}, @@ -1862,10 +1804,6 @@ toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] -tomli = [ - {file = "tomli-1.2.1-py3-none-any.whl", hash = "sha256:8dd0e9524d6f386271a36b41dbf6c57d8e32fd96fd22b6584679dc569d20899f"}, - {file = "tomli-1.2.1.tar.gz", hash = "sha256:a5b75cb6f3968abb47af1b40c1819dc519ea82bcc065776a866e8d74c5ca9442"}, -] toolz = [ {file = "toolz-0.11.1-py3-none-any.whl", hash = "sha256:1bc473acbf1a1db4e72a1ce587be347450e8f08324908b8a266b486f408f04d5"}, {file = "toolz-0.11.1.tar.gz", hash = "sha256:c7a47921f07822fe534fb1c01c9931ab335a4390c782bd28c6bcc7c2f71f3fbf"}, diff --git a/pyproject.toml b/pyproject.toml index e4275be..797a99c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ pytest-sugar = "^0.9.4" pytest-cov = "^2.12.1" coverage = "^5.5" alembic = "^1.6.5" -black = "^21.7b0" CProfileV = "^1.0.7" regex = "^2021.10.8"