Weave trace db through to fetch

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ repos:
 -   repo: local
     hooks:
     - id: system
-      name: Black
+      name: black
       entry: poetry run black .
       pass_filenames: false
       language: system

cli.py

@@ -6,7 +6,7 @@ import click
 from web3 import Web3
 
 from mev_inspect.classifiers.trace import TraceClassifier
-from mev_inspect.db import get_inspect_session
+from mev_inspect.db import get_inspect_session, get_trace_session
 from mev_inspect.inspect_block import inspect_block
 from mev_inspect.provider import get_base_provider
 
@@ -27,7 +27,9 @@ def cli():
 @click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
 @click.option("--cache/--no-cache", default=True)
 def inspect_block_command(block_number: int, rpc: str, cache: bool):
-    db_session = get_inspect_session()
+    inspect_db_session = get_inspect_session()
+    trace_db_session = get_trace_session()
+
     base_provider = get_base_provider(rpc)
     w3 = Web3(base_provider)
     trace_classifier = TraceClassifier()
@@ -36,12 +38,13 @@ def inspect_block_command(block_number: int, rpc: str, cache: bool):
         logger.info("Skipping cache")
 
     inspect_block(
-        db_session,
+        inspect_db_session,
         base_provider,
         w3,
         trace_classifier,
         block_number,
         should_cache=cache,
+        trace_db_session=trace_db_session,
     )
 
 
@@ -54,7 +57,9 @@ def inspect_many_blocks_command(
     after_block: int, before_block: int, rpc: str, cache: bool
 ):
 
-    db_session = get_inspect_session()
+    inspect_db_session = get_inspect_session()
+    trace_db_session = get_trace_session()
+
     base_provider = get_base_provider(rpc)
     w3 = Web3(base_provider)
     trace_classifier = TraceClassifier()
@@ -72,13 +77,14 @@ def inspect_many_blocks_command(
         logger.info(dashes)
 
         inspect_block(
-            db_session,
+            inspect_db_session,
             base_provider,
             w3,
             trace_classifier,
             block_number,
             should_write_classified_traces=False,
             should_cache=cache,
+            trace_db_session=trace_db_session,
         )
 
 

mev_inspect/block.py

@@ -1,6 +1,7 @@
 from pathlib import Path
-from typing import List
+from typing import List, Optional
 
+from sqlalchemy import orm
 from web3 import Web3
 
 from mev_inspect.fees import fetch_base_fee_per_gas
@@ -16,28 +17,36 @@ def get_latest_block_number(w3: Web3) -> int:
 
 
 def create_from_block_number(
-    base_provider, w3: Web3, block_number: int, should_cache: bool
+    base_provider,
+    w3: Web3,
+    block_number: int,
+    should_cache: bool,
+    trace_db_session: Optional[orm.Session],
 ) -> Block:
     if not should_cache:
-        return fetch_block(w3, base_provider, block_number)
+        return fetch_block(w3, base_provider, block_number, trace_db_session)
 
     cache_path = _get_cache_path(block_number)
 
     if cache_path.is_file():
         print(f"Cache for block {block_number} exists, " "loading data from cache")
-
         return Block.parse_file(cache_path)
     else:
         print(f"Cache for block {block_number} did not exist, getting data")
 
-        block = fetch_block(w3, base_provider, block_number)
+        block = fetch_block(w3, base_provider, block_number, trace_db_session)
 
         cache_block(cache_path, block)
 
         return block
 
 
-def fetch_block(w3, base_provider, block_number: int) -> Block:
+def fetch_block(
+    w3,
+    base_provider,
+    block_number: int,
+    trace_db_session: Optional[orm.Session],  # pylint: disable=unused-argument
+) -> Block:
     block_json = w3.eth.get_block(block_number)
     receipts_json = base_provider.make_request("eth_getBlockReceipts", [block_number])
     traces_json = w3.parity.trace_block(block_number)

mev_inspect/inspect_block.py

@@ -1,5 +1,7 @@
 import logging
+from typing import Optional
 
+from sqlalchemy import orm
 from web3 import Web3
 
 from mev_inspect.arbitrages import get_arbitrages
@@ -34,12 +36,13 @@ logger = logging.getLogger(__name__)
 
 
 def inspect_block(
-    db_session,
+    inspect_db_session: orm.Session,
     base_provider,
     w3: Web3,
     trace_clasifier: TraceClassifier,
     block_number: int,
     should_cache: bool,
+    trace_db_session: Optional[orm.Session],
     should_write_classified_traces: bool = True,
 ):
     block = create_from_block_number(
@@ -47,6 +50,7 @@ def inspect_block(
         w3,
         block_number,
         should_cache,
+        trace_db_session,
     )
 
     logger.info(f"Total traces: {len(block.traces)}")
@@ -60,36 +64,36 @@ def inspect_block(
     logger.info(f"Returned {len(classified_traces)} classified traces")
 
     if should_write_classified_traces:
-        delete_classified_traces_for_block(db_session, block_number)
+        delete_classified_traces_for_block(inspect_db_session, block_number)
-        write_classified_traces(db_session, classified_traces)
+        write_classified_traces(inspect_db_session, classified_traces)
 
     transfers = get_transfers(classified_traces)
     logger.info(f"Found {len(transfers)} transfers")
 
-    delete_transfers_for_block(db_session, block_number)
+    delete_transfers_for_block(inspect_db_session, block_number)
-    write_transfers(db_session, transfers)
+    write_transfers(inspect_db_session, transfers)
 
     swaps = get_swaps(classified_traces)
     logger.info(f"Found {len(swaps)} swaps")
 
-    delete_swaps_for_block(db_session, block_number)
+    delete_swaps_for_block(inspect_db_session, block_number)
-    write_swaps(db_session, swaps)
+    write_swaps(inspect_db_session, swaps)
 
     arbitrages = get_arbitrages(swaps)
     logger.info(f"Found {len(arbitrages)} arbitrages")
 
-    delete_arbitrages_for_block(db_session, block_number)
+    delete_arbitrages_for_block(inspect_db_session, block_number)
-    write_arbitrages(db_session, arbitrages)
+    write_arbitrages(inspect_db_session, arbitrages)
 
     liquidations = get_liquidations(classified_traces)
     logger.info(f"Found {len(liquidations)} liquidations")
 
-    delete_liquidations_for_block(db_session, block_number)
+    delete_liquidations_for_block(inspect_db_session, block_number)
-    write_liquidations(db_session, liquidations)
+    write_liquidations(inspect_db_session, liquidations)
 
     miner_payments = get_miner_payments(
         block.miner, block.base_fee_per_gas, classified_traces, block.receipts
     )
 
-    delete_miner_payments_for_block(db_session, block_number)
+    delete_miner_payments_for_block(inspect_db_session, block_number)
-    write_miner_payments(db_session, miner_payments)
+    write_miner_payments(inspect_db_session, miner_payments)

poetry.lock

@@ -31,14 +31,6 @@ python-dateutil = "*"
 python-editor = ">=0.3"
 SQLAlchemy = ">=1.3.0"
 
-[[package]]
-name = "appdirs"
-version = "1.4.4"
-description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
-category = "dev"
-optional = false
-python-versions = "*"
-
 [[package]]
 name = "astroid"
 version = "2.7.2"
@@ -112,28 +104,6 @@ category = "main"
 optional = false
 python-versions = "*"
 
-[[package]]
-name = "black"
-version = "21.7b0"
-description = "The uncompromising code formatter."
-category = "dev"
-optional = false
-python-versions = ">=3.6.2"
-
-[package.dependencies]
-appdirs = "*"
-click = ">=7.1.2"
-mypy-extensions = ">=0.4.3"
-pathspec = ">=0.8.1,<1"
-regex = ">=2020.1.8"
-tomli = ">=0.2.6,<2.0.0"
-
-[package.extras]
-colorama = ["colorama (>=0.4.3)"]
-d = ["aiohttp (>=3.6.0)", "aiohttp-cors (>=0.4.0)"]
-python2 = ["typed-ast (>=1.4.2)"]
-uvloop = ["uvloop (>=0.15.2)"]
-
 [[package]]
 name = "bottle"
 version = "0.12.19"
@@ -625,14 +595,6 @@ python-versions = "*"
 [package.dependencies]
 six = ">=1.9.0"
 
-[[package]]
-name = "pathspec"
-version = "0.9.0"
-description = "Utility library for gitignore style pattern matching of file paths."
-category = "dev"
-optional = false
-python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
-
 [[package]]
 name = "platformdirs"
 version = "2.2.0"
@@ -938,14 +900,6 @@ category = "dev"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
 
-[[package]]
-name = "tomli"
-version = "1.2.1"
-description = "A lil' TOML parser"
-category = "dev"
-optional = false
-python-versions = ">=3.6"
-
 [[package]]
 name = "toolz"
 version = "0.11.1"
@@ -1063,7 +1017,7 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.9"
-content-hash = "7b8511360be2f67f85c4e9e2d5ee73da2ce14cc691a44e8c6cb03552af6d75b5"
+content-hash = "baade6f62f3adaff192b2c85b4f602f4990b9b99d6fcce904aeb5087b6fa1921"
 
 [metadata.files]
 aiohttp = [
@@ -1109,10 +1063,6 @@ alembic = [
     {file = "alembic-1.6.5-py2.py3-none-any.whl", hash = "sha256:e78be5b919f5bb184e3e0e2dd1ca986f2362e29a2bc933c446fe89f39dbe4e9c"},
     {file = "alembic-1.6.5.tar.gz", hash = "sha256:a21fedebb3fb8f6bbbba51a11114f08c78709377051384c9c5ead5705ee93a51"},
 ]
-appdirs = [
-    {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
-    {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
-]
 astroid = [
     {file = "astroid-2.7.2-py3-none-any.whl", hash = "sha256:ecc50f9b3803ebf8ea19aa2c6df5622d8a5c31456a53c741d3be044d96ff0948"},
     {file = "astroid-2.7.2.tar.gz", hash = "sha256:b6c2d75cd7c2982d09e7d41d70213e863b3ba34d3bd4014e08f167cee966e99e"},
@@ -1140,10 +1090,6 @@ base58 = [
 bitarray = [
     {file = "bitarray-1.2.2.tar.gz", hash = "sha256:27a69ffcee3b868abab3ce8b17c69e02b63e722d4d64ffd91d659f81e9984954"},
 ]
-black = [
-    {file = "black-21.7b0-py3-none-any.whl", hash = "sha256:1c7aa6ada8ee864db745b22790a32f94b2795c253a75d6d9b5e439ff10d23116"},
-    {file = "black-21.7b0.tar.gz", hash = "sha256:c8373c6491de9362e39271630b65b964607bc5c79c83783547d76c839b3aa219"},
-]
 bottle = [
     {file = "bottle-0.12.19-py3-none-any.whl", hash = "sha256:f6b8a34fe9aa406f9813c02990db72ca69ce6a158b5b156d2c41f345016a723d"},
     {file = "bottle-0.12.19.tar.gz", hash = "sha256:a9d73ffcbc6a1345ca2d7949638db46349f5b2b77dac65d6494d45c23628da2c"},
@@ -1550,10 +1496,6 @@ packaging = [
 parsimonious = [
     {file = "parsimonious-0.8.1.tar.gz", hash = "sha256:3add338892d580e0cb3b1a39e4a1b427ff9f687858fdd61097053742391a9f6b"},
 ]
-pathspec = [
-    {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"},
-    {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"},
-]
 platformdirs = [
     {file = "platformdirs-2.2.0-py3-none-any.whl", hash = "sha256:4666d822218db6a262bdfdc9c39d21f23b4cfdb08af331a81e92751daf6c866c"},
     {file = "platformdirs-2.2.0.tar.gz", hash = "sha256:632daad3ab546bd8e6af0537d09805cec458dce201bccfe23012df73332e181e"},
@@ -1862,10 +1804,6 @@ toml = [
     {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
     {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
 ]
-tomli = [
-    {file = "tomli-1.2.1-py3-none-any.whl", hash = "sha256:8dd0e9524d6f386271a36b41dbf6c57d8e32fd96fd22b6584679dc569d20899f"},
-    {file = "tomli-1.2.1.tar.gz", hash = "sha256:a5b75cb6f3968abb47af1b40c1819dc519ea82bcc065776a866e8d74c5ca9442"},
-]
 toolz = [
     {file = "toolz-0.11.1-py3-none-any.whl", hash = "sha256:1bc473acbf1a1db4e72a1ce587be347450e8f08324908b8a266b486f408f04d5"},
     {file = "toolz-0.11.1.tar.gz", hash = "sha256:c7a47921f07822fe534fb1c01c9931ab335a4390c782bd28c6bcc7c2f71f3fbf"},

pyproject.toml

@@ -22,7 +22,6 @@ pytest-sugar = "^0.9.4"
 pytest-cov = "^2.12.1"
 coverage = "^5.5"
 alembic = "^1.6.5"
-black = "^21.7b0"
 CProfileV = "^1.0.7"
 regex = "^2021.10.8"