Merge pull request #1 from marlinprotocol/main-gethmerge

Fetch master

README.md

@@ -35,6 +35,10 @@ kind create cluster
 
 Set an environment variable `RPC_URL` to an RPC for fetching blocks.
 
+mev-inspect-py currently requires a node with support for Erigon traces and receipts (not geth yet 😔).
+
+[pokt.network](pokt.network)'s "Ethereum Mainnet Archival with trace calls" is a good hosted option.
+
 Example:
 
 ```
@@ -54,7 +58,7 @@ Press "space" to see a browser of the services starting up.
 On first startup, you'll need to apply database migrations with:
 
 ```
-kubectl exec deploy/mev-inspect -- alembic upgrade head
+./mev exec alembic upgrade head
 ```
 
 ## Usage
@@ -65,7 +69,7 @@ Inspecting block [12914944](https://twitter.com/mevalphaleak/status/142041643757
 **Note**: Add `--geth` at the end if RPC_URL points to a geth / geth like node.
 
 ```
-kubectl exec deploy/mev-inspect -- poetry run inspect-block 12914944
+./mev inspect 12914944
 ```
 
 ### Inspect many blocks
@@ -74,7 +78,7 @@ Inspecting blocks 12914944 to 12914954:
 **Note**: Add `--geth` at the end if RPC_URL points to a geth / geth like node.
 
 ```
-kubectl exec deploy/mev-inspect -- poetry run inspect-many-blocks 12914944 12914954
+./mev inspect-many 12914944 12914954
 ```
 
 ### Inspect all incoming blocks
@@ -82,24 +86,46 @@ kubectl exec deploy/mev-inspect -- poetry run inspect-many-blocks 12914944 12914
 Start a block listener with:
 
 ```
-kubectl exec deploy/mev-inspect -- /app/listener start
+./mev listener start
 ```
 
 By default, it will pick up wherever you left off.
 If running for the first time, listener starts at the latest block.
 
-See logs for the listener with:
+Tail logs for the listener with:
 
 ```
-kubectl exec deploy/mev-inspect -- tail -f listener.log
+./mev listener tail
 ```
 
 And stop the listener with:
 
 ```
-kubectl exec deploy/mev-inspect -- /app/listener stop
+./mev listener stop
 ```
 
+### Backfilling
+
+For larger backfills, you can inspect many blocks in parallel using kubernetes
+
+To inspect blocks 12914944 to 12915044 divided across 10 worker pods:
+```
+./mev backfill 12914944 12915044 10
+```
+
+You can see worker pods spin up then complete by watching the status of all pods
+```
+watch kubectl get pods
+```
+
+To watch the logs for a given pod, take its pod name using the above, then run:
+```
+kubectl logs -f pod/mev-inspect-backfill-abcdefg
+```
+
+(where `mev-inspect-backfill-abcdefg` is your actual pod name)
+
+
 ### Exploring
 
 All inspect output data is stored in Postgres.
@@ -107,7 +133,7 @@ All inspect output data is stored in Postgres.
 To connect to the local Postgres database for querying, launch a client container with:
 
 ```
-kubectl run -i --rm --tty postgres-client --env="PGPASSWORD=password" --image=jbergknoff/postgresql-client -- mev_inspect --host=postgresql --user=postgres
+./mev db
 ```
 
 When you see the prompt:
@@ -161,7 +187,7 @@ tilt up
 And rerun migrations to create the tables again:
 
 ```
-kubectl exec deploy/mev-inspect -- alembic upgrade head
+./mev exec alembic upgrade head
 ```
 
 ### I was using the docker-compose setup and want to switch to kube, now what?

Tiltfile

@@ -13,6 +13,10 @@ k8s_yaml(configmap_from_dict("mev-inspect-rpc", inputs = {
     "url" : os.environ["RPC_URL"],
 }))
 
+k8s_yaml(configmap_from_dict("mev-inspect-listener-healthcheck", inputs = {
+    "url" : os.getenv("LISTENER_HEALTHCHECK_URL", default=""),
+}))
+
 k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = {
     "username" : "postgres",
     "password": "password",
@@ -36,3 +40,9 @@ docker_build_with_restart("mev-inspect-py", ".",
 )
 k8s_yaml(helm('./k8s/mev-inspect', name='mev-inspect'))
 k8s_resource(workload="mev-inspect", resource_deps=["postgresql-postgresql"])
+
+local_resource(
+    'pg-port-forward',
+    serve_cmd='kubectl port-forward --namespace default svc/postgresql 5432:5432',
+    resource_deps=["postgresql-postgresql"]
+)

alembic/versions/04a3bb3740c3_change_miner_payments_and_transfers_.py

@@ -0,0 +1,55 @@
+"""Change miner payments and transfers primary keys to include block number
+
+Revision ID: 04a3bb3740c3
+Revises: a10d68643476
+Create Date: 2021-11-02 22:42:01.702538
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "04a3bb3740c3"
+down_revision = "a10d68643476"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # transfers
+    op.execute("ALTER TABLE transfers DROP CONSTRAINT transfers_pkey")
+    op.create_primary_key(
+        "transfers_pkey",
+        "transfers",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+    op.drop_index("ix_transfers_block_number")
+
+    # miner_payments
+    op.execute("ALTER TABLE miner_payments DROP CONSTRAINT miner_payments_pkey")
+    op.create_primary_key(
+        "miner_payments_pkey",
+        "miner_payments",
+        ["block_number", "transaction_hash"],
+    )
+    op.drop_index("ix_block_number")
+
+
+def downgrade():
+    # transfers
+    op.execute("ALTER TABLE transfers DROP CONSTRAINT transfers_pkey")
+    op.create_index("ix_transfers_block_number", "transfers", ["block_number"])
+    op.create_primary_key(
+        "transfers_pkey",
+        "transfers",
+        ["transaction_hash", "trace_address"],
+    )
+
+    # miner_payments
+    op.execute("ALTER TABLE miner_payments DROP CONSTRAINT miner_payments_pkey")
+    op.create_index("ix_block_number", "miner_payments", ["block_number"])
+    op.create_primary_key(
+        "miner_payments_pkey",
+        "miner_payments",
+        ["transaction_hash"],
+    )

alembic/versions/0cef835f7b36_rename_pool_address_to_contract_address.py

@@ -0,0 +1,27 @@
+"""Rename pool_address to contract_address
+
+Revision ID: 0cef835f7b36
+Revises: 5427d62a2cc0
+Create Date: 2021-11-19 15:36:15.152622
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "0cef835f7b36"
+down_revision = "5427d62a2cc0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column(
+        "swaps", "pool_address", nullable=False, new_column_name="contract_address"
+    )
+
+
+def downgrade():
+    op.alter_column(
+        "swaps", "contract_address", nullable=False, new_column_name="pool_address"
+    )

alembic/versions/2c90b2b8a80b_add_blocks_table.py

@@ -0,0 +1,29 @@
+"""Add blocks table
+
+Revision ID: 2c90b2b8a80b
+Revises: 04a3bb3740c3
+Create Date: 2021-11-17 18:29:13.065944
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "2c90b2b8a80b"
+down_revision = "04a3bb3740c3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "blocks",
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("block_timestamp", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("block_number"),
+    )
+
+
+def downgrade():
+    op.drop_table("blocks")

alembic/versions/3417f49d97b3_cahnge_swap_primary_key_to_include_.py

@@ -0,0 +1,46 @@
+"""Cahnge swap primary key to include block number
+
+Revision ID: 3417f49d97b3
+Revises: 205ce02374b3
+Create Date: 2021-11-02 20:50:32.854996
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "3417f49d97b3"
+down_revision = "205ce02374b3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("ALTER TABLE swaps DROP CONSTRAINT swaps_pkey CASCADE")
+    op.create_primary_key(
+        "swaps_pkey",
+        "swaps",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+    op.create_index(
+        "arbitrage_swaps_swaps_idx",
+        "arbitrage_swaps",
+        ["swap_transaction_hash", "swap_trace_address"],
+    )
+
+
+def downgrade():
+    op.drop_index("arbitrage_swaps_swaps_idx")
+    op.execute("ALTER TABLE swaps DROP CONSTRAINT swaps_pkey CASCADE")
+    op.create_primary_key(
+        "swaps_pkey",
+        "swaps",
+        ["transaction_hash", "trace_address"],
+    )
+    op.create_foreign_key(
+        "arbitrage_swaps_swaps_fkey",
+        "arbitrage_swaps",
+        "swaps",
+        ["swap_transaction_hash", "swap_trace_address"],
+        ["transaction_hash", "trace_address"],
+    )

alembic/versions/5427d62a2cc0_change_transfers_trace_address_to_array.py

@@ -0,0 +1,47 @@
+"""Change transfers trace address to ARRAY
+
+Revision ID: 5427d62a2cc0
+Revises: d540242ae368
+Create Date: 2021-11-19 13:25:11.252774
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "5427d62a2cc0"
+down_revision = "d540242ae368"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_constraint("transfers_pkey", "transfers")
+    op.alter_column(
+        "transfers",
+        "trace_address",
+        type_=sa.ARRAY(sa.Integer),
+        nullable=False,
+        postgresql_using="trace_address::int[]",
+    )
+    op.create_primary_key(
+        "transfers_pkey",
+        "transfers",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+
+
+def downgrade():
+    op.drop_constraint("transfers_pkey", "transfers")
+    op.alter_column(
+        "transfers",
+        "trace_address",
+        type_=sa.String(256),
+        nullable=False,
+    )
+    op.create_primary_key(
+        "transfers_pkey",
+        "transfers",
+        ["block_number", "transaction_hash", "trace_address"],
+    )

alembic/versions/a10d68643476_change_classified_traces_primary_key_to_.py

@@ -0,0 +1,35 @@
+"""Change classified traces primary key to include block number
+
+Revision ID: a10d68643476
+Revises: 3417f49d97b3
+Create Date: 2021-11-02 22:03:26.312317
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "a10d68643476"
+down_revision = "3417f49d97b3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("ALTER TABLE classified_traces DROP CONSTRAINT classified_traces_pkey")
+    op.create_primary_key(
+        "classified_traces_pkey",
+        "classified_traces",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+    op.drop_index("i_block_number")
+
+
+def downgrade():
+    op.execute("ALTER TABLE classified_traces DROP CONSTRAINT classified_traces_pkey")
+    op.create_index("i_block_number", "classified_traces", ["block_number"])
+    op.create_primary_key(
+        "classified_traces_pkey",
+        "classified_traces",
+        ["transaction_hash", "trace_address"],
+    )

alembic/versions/d540242ae368_create_usd_prices_table.py

@@ -0,0 +1,30 @@
+"""Create usd_prices table
+
+Revision ID: d540242ae368
+Revises: 2c90b2b8a80b
+Create Date: 2021-11-18 04:30:06.802857
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "d540242ae368"
+down_revision = "2c90b2b8a80b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "prices",
+        sa.Column("timestamp", sa.TIMESTAMP),
+        sa.Column("usd_price", sa.Numeric, nullable=False),
+        sa.Column("token_address", sa.String(256), nullable=False),
+        sa.PrimaryKeyConstraint("token_address", "timestamp"),
+    )
+
+
+def downgrade():
+    op.drop_table("prices")

cli.py

@@ -1,21 +1,18 @@
-import os
 import logging
+import os
 import sys
 
 import click
 from web3 import Web3
 from web3.middleware import geth_poa_middleware
 
-from mev_inspect.classifiers.trace import TraceClassifier
+from mev_inspect.concurrency import coro
 from mev_inspect.db import get_inspect_session, get_trace_session
-from mev_inspect.inspect_block import inspect_block
+from mev_inspect.inspector import MEVInspector
-from mev_inspect.provider import get_base_provider
-
 
 RPC_URL_ENV = "RPC_URL"
 
 logging.basicConfig(stream=sys.stdout, level=logging.INFO)
-logger = logging.getLogger(__name__)
 
 
 @click.group()
@@ -26,73 +23,66 @@ def cli():
 @cli.command()
 @click.argument("block_number", type=int)
 @click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
-@click.option("--cache/--no-cache", default=True)
 @click.option("--geth/--no-geth", default=False)
-def inspect_block_command(block_number: int, rpc: str, cache: bool, geth: bool):
+@coro
+async def inspect_block_command(block_number: int, rpc: str, geth: bool):
     inspect_db_session = get_inspect_session()
     trace_db_session = get_trace_session()
 
-    base_provider = get_base_provider(rpc)
+    inspector = MEVInspector(rpc, inspect_db_session, trace_db_session, geth)
-    w3 = Web3(base_provider)
+    await inspector.inspect_single_block(block=block_number)
-    if geth:
-        w3.middleware_onion.inject(geth_poa_middleware, layer=0)
-    trace_classifier = TraceClassifier()
 
-    if not cache:
-        logger.info("Skipping cache")
 
-    inspect_block(
+@cli.command()
-        inspect_db_session,
+@click.argument("block_number", type=int)
-        base_provider,
+@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
-        w3,
+@coro
-        geth,
+async def fetch_block_command(block_number: int, rpc: str):
-        trace_classifier,
+    inspect_db_session = get_inspect_session()
-        block_number,
+    trace_db_session = get_trace_session()
-        trace_db_session=trace_db_session,
+
-    )
+    inspector = MEVInspector(rpc, inspect_db_session, trace_db_session, false)
+    block = await inspector.create_from_block(block_number=block_number)
+    print(block.json())
 
 
 @cli.command()
 @click.argument("after_block", type=int)
 @click.argument("before_block", type=int)
 @click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
-@click.option("--cache/--no-cache", default=True)
 @click.option("--geth/--no-geth", default=False)
-def inspect_many_blocks_command(
-    after_block: int, before_block: int, rpc: str, cache: bool, geth: bool
-):
 
+@click.option(
+    "--max-concurrency",
+    type=int,
+    help="maximum number of concurrent connections",
+    default=5,
+)
+@click.option(
+    "--request-timeout", type=int, help="timeout for requests to nodes", default=500
+)
+@coro
+async def inspect_many_blocks_command(
+    after_block: int,
+    before_block: int,
+    rpc: str,
+    max_concurrency: int,
+    request_timeout: int,
+    geth: bool
+):
     inspect_db_session = get_inspect_session()
     trace_db_session = get_trace_session()
-
+    inspector = MEVInspector(
-    base_provider = get_base_provider(rpc)
+        rpc,
-    w3 = Web3(base_provider)
+        inspect_db_session,
-    if geth:
+        trace_db_session,
-        w3.middleware_onion.inject(geth_poa_middleware, layer=0)
+        max_concurrency=max_concurrency,
-    trace_classifier = TraceClassifier()
+        request_timeout=request_timeout,
-
+        geth
-    if not cache:
+    )
-        logger.info("Skipping cache")
+    await inspector.inspect_many_blocks(
-
+        after_block=after_block, before_block=before_block
-    for i, block_number in enumerate(range(after_block, before_block)):
+    )
-        block_message = (
-            f"Running for {block_number} ({i+1}/{before_block - after_block})"
-        )
-        dashes = "-" * len(block_message)
-        logger.info(dashes)
-        logger.info(block_message)
-        logger.info(dashes)
-
-        inspect_block(
-            inspect_db_session,
-            base_provider,
-            w3,
-            geth,
-            trace_classifier,
-            block_number,
-            trace_db_session=trace_db_session,
-        )
-
 
 def get_rpc_url() -> str:
     return os.environ["RPC_URL"]

k8s/mev-inspect-backfill/templates/job.yaml

@@ -43,6 +43,24 @@ spec:
               secretKeyRef:
                 name: mev-inspect-db-credentials
                 key: password
+          - name: TRACE_DB_HOST
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: host
+                optional: true
+          - name: TRACE_DB_USER
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: username
+                optional: true
+          - name: TRACE_DB_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: password
+                optional: true
           - name: RPC_URL
             valueFrom:
               configMapKeyRef:

k8s/mev-inspect/templates/deployment.yaml

@@ -78,6 +78,12 @@ spec:
               configMapKeyRef:
                 name: mev-inspect-rpc
                 key: url
+          - name: LISTENER_HEALTHCHECK_URL
+            valueFrom:
+              configMapKeyRef:
+                name: mev-inspect-listener-healthcheck
+                key: url
+                optional: true
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

listener

@@ -25,6 +25,9 @@ case "$1" in
 	start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE
         echo "."
 	;;
+  tail)
+        tail -f listener.log
+	;;
   restart)
         echo -n "Restarting daemon: "$NAME
 	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile $PIDFILE
@@ -40,7 +43,7 @@ case "$1" in
     ;;
 
   *)
-	echo "Usage: "$1" {start|stop|restart}"
+	echo "Usage: "$1" {start|stop|restart|tail}"
 	exit 1
 esac
 

listener.py

@@ -1,78 +1,97 @@
+import asyncio
 import logging
 import os
-import time
 
-from web3 import Web3
+import aiohttp
 
 from mev_inspect.block import get_latest_block_number
+from mev_inspect.concurrency import coro
 from mev_inspect.crud.latest_block_update import (
     find_latest_block_update,
     update_latest_block,
 )
-from mev_inspect.classifiers.trace import TraceClassifier
 from mev_inspect.db import get_inspect_session, get_trace_session
-from mev_inspect.inspect_block import inspect_block
+from mev_inspect.inspector import MEVInspector
 from mev_inspect.provider import get_base_provider
 from mev_inspect.signal_handler import GracefulKiller
 
 
-logging.basicConfig(filename="listener.log", level=logging.INFO)
+logging.basicConfig(filename="listener.log", filemode="a", level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 # lag to make sure the blocks we see are settled
 BLOCK_NUMBER_LAG = 5
 
 
-def run():
+@coro
+async def run():
     rpc = os.getenv("RPC_URL")
     if rpc is None:
         raise RuntimeError("Missing environment variable RPC_URL")
 
+    healthcheck_url = os.getenv("LISTENER_HEALTHCHECK_URL")
+
     logger.info("Starting...")
 
     killer = GracefulKiller()
 
     inspect_db_session = get_inspect_session()
     trace_db_session = get_trace_session()
-    trace_classifier = TraceClassifier()
 
+    inspector = MEVInspector(rpc, inspect_db_session, trace_db_session)
     base_provider = get_base_provider(rpc)
-    w3 = Web3(base_provider)
-
-    latest_block_number = get_latest_block_number(w3)
 
     while not killer.kill_now:
-        last_written_block = find_latest_block_update(inspect_db_session)
+        await inspect_next_block(
-        logger.info(f"Latest block: {latest_block_number}")
+            inspector,
-        logger.info(f"Last written block: {last_written_block}")
+            inspect_db_session,
-
+            base_provider,
-        if (last_written_block is None) or (
+            healthcheck_url,
-            last_written_block < (latest_block_number - BLOCK_NUMBER_LAG)
+        )
-        ):
-            block_number = (
-                latest_block_number
-                if last_written_block is None
-                else last_written_block + 1
-            )
-
-            logger.info(f"Writing block: {block_number}")
-
-            inspect_block(
-                inspect_db_session,
-                base_provider,
-                w3,
-                trace_classifier,
-                block_number,
-                trace_db_session=trace_db_session,
-            )
-            update_latest_block(inspect_db_session, block_number)
-        else:
-            time.sleep(5)
-            latest_block_number = get_latest_block_number(w3)
 
     logger.info("Stopping...")
 
 
+async def inspect_next_block(
+    inspector: MEVInspector,
+    inspect_db_session,
+    base_provider,
+    healthcheck_url,
+):
+    latest_block_number = await get_latest_block_number(base_provider)
+    last_written_block = find_latest_block_update(inspect_db_session)
+
+    logger.info(f"Latest block: {latest_block_number}")
+    logger.info(f"Last written block: {last_written_block}")
+
+    if last_written_block is None:
+        # maintain lag if no blocks written yet
+        last_written_block = latest_block_number - 1
+
+    if last_written_block < (latest_block_number - BLOCK_NUMBER_LAG):
+        block_number = (
+            latest_block_number
+            if last_written_block is None
+            else last_written_block + 1
+        )
+
+        logger.info(f"Writing block: {block_number}")
+
+        await inspector.inspect_single_block(block=block_number)
+        update_latest_block(inspect_db_session, block_number)
+
+        if healthcheck_url:
+            await ping_healthcheck_url(healthcheck_url)
+    else:
+        await asyncio.sleep(5)
+
+
+async def ping_healthcheck_url(url):
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url):
+            pass
+
+
 if __name__ == "__main__":
     try:
         run()

mev

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 set -e
 
@@ -24,6 +24,9 @@ case "$1" in
         echo "Connecting to $DB_NAME"
         db
 	;;
+  listener)
+        kubectl exec -ti deploy/mev-inspect -- ./listener $2
+	;;
   backfill)
         start_block_number=$2
         end_block_number=$3
@@ -37,12 +40,28 @@ case "$1" in
         echo "Inspecting block $block_number"
         kubectl exec -ti deploy/mev-inspect -- poetry run inspect-block $block_number
 	;;
+  inspect-many)
+        start_block_number=$2
+        end_block_number=$3
+        echo "Inspecting from block $start_block_number to $end_block_number"
+        kubectl exec -ti deploy/mev-inspect -- \
+            poetry run inspect-many-blocks $start_block_number $end_block_number
+	;;
   test)
         echo "Running tests"
         kubectl exec -ti deploy/mev-inspect -- poetry run pytest tests
 	;;
+  fetch)
+      block_number=$2
+      echo "Fetching block $block_number"
+      kubectl exec -ti deploy/mev-inspect -- poetry run fetch-block $block_number
+  ;;
+  exec)
+        shift
+        kubectl exec -ti deploy/mev-inspect -- $@
+  ;;
   *)
-	echo "Usage: "$1" {inspect|test}"
+	echo "Usage: "$1" {db|backfill|inspect|test}"
 	exit 1
 esac
 

mev_inspect/aave_liquidations.py

@@ -4,8 +4,9 @@ from mev_inspect.traces import (
     get_child_traces,
     is_child_of_any_address,
 )
-from mev_inspect.schemas.classified_traces import (
+from mev_inspect.schemas.traces import (
     ClassifiedTrace,
+    CallTrace,
     DecodedCallTrace,
     Classification,
     Protocol,
@@ -77,6 +78,7 @@ def get_aave_liquidations(
                     block_number=trace.block_number,
                 )
             )
+
     return liquidations
 
 
@@ -88,17 +90,17 @@ def _get_payback_token_and_amount(
 
     for child in child_traces:
 
-        if child.classification == Classification.transfer and isinstance(
+        if isinstance(child, CallTrace):
-            child, DecodedCallTrace
-        ):
 
             child_transfer: Optional[Transfer] = get_transfer(child)
 
-            if (
+            if child_transfer is not None:
-                child_transfer is not None
+
-                and child_transfer.to_address == liquidator
+                if (
-                and child.from_address in AAVE_CONTRACT_ADDRESSES
+                    child_transfer.to_address == liquidator
-            ):
+                    and child.from_address in AAVE_CONTRACT_ADDRESSES
-                return child_transfer.token_address, child_transfer.amount
+                ):
+
+                    return child_transfer.token_address, child_transfer.amount
 
     return liquidation.inputs["_collateral"], 0

mev_inspect/abi.py

@@ -4,8 +4,8 @@ from typing import Optional
 
 from pydantic import parse_obj_as
 
-from mev_inspect.schemas import ABI
+from mev_inspect.schemas.abi import ABI
-from mev_inspect.schemas.classified_traces import Protocol
+from mev_inspect.schemas.traces import Protocol
 
 
 THIS_FILE_DIRECTORY = Path(__file__).parents[0]

mev_inspect/arbitrages.py

@@ -1,5 +1,5 @@
 from itertools import groupby
-from typing import List, Optional
+from typing import List, Tuple
 
 from mev_inspect.schemas.arbitrages import Arbitrage
 from mev_inspect.schemas.swaps import Swap
@@ -23,70 +23,111 @@ def get_arbitrages(swaps: List[Swap]) -> List[Arbitrage]:
 
 
 def _get_arbitrages_from_swaps(swaps: List[Swap]) -> List[Arbitrage]:
-    pool_addresses = {swap.pool_address for swap in swaps}
+    """
+    An arbitrage is defined as multiple swaps in a series that result in the initial token being returned
+    to the initial sender address.
+
+    There are 2 types of swaps that are most common (99%+).
+    Case I (fully routed):
+    BOT -> A/B -> B/C -> C/A -> BOT
+
+    Case II (always return to bot):
+    BOT -> A/B -> BOT -> B/C -> BOT -> A/C -> BOT
+
+    There is only 1 correct way to route Case I, but for Case II the following valid routes could be found:
+    A->B->C->A / B->C->A->B / C->A->B->C. Thus when multiple valid routes are found we filter to the set that
+    happen in valid order.
+    """
 
     all_arbitrages = []
 
-    for index, first_swap in enumerate(swaps):
+    start_ends = _get_all_start_end_swaps(swaps)
-        other_swaps = swaps[:index] + swaps[index + 1 :]
+    if len(start_ends) == 0:
+        return []
 
-        if first_swap.from_address not in pool_addresses:
+    # for (start, end) in filtered_start_ends:
-            arbitrage = _get_arbitrage_starting_with_swap(first_swap, other_swaps)
+    for (start, end) in start_ends:
+        potential_intermediate_swaps = [
+            swap for swap in swaps if swap is not start and swap is not end
+        ]
+        routes = _get_all_routes(start, end, potential_intermediate_swaps)
 
-            if arbitrage is not None:
+        for route in routes:
-                all_arbitrages.append(arbitrage)
+            start_amount = route[0].token_in_amount
-
+            end_amount = route[-1].token_out_amount
-    return all_arbitrages
-
-
-def _get_arbitrage_starting_with_swap(
-    start_swap: Swap,
-    other_swaps: List[Swap],
-) -> Optional[Arbitrage]:
-    swap_path = [start_swap]
-    current_swap: Swap = start_swap
-
-    while True:
-        next_swap = _get_swap_from_address(
-            current_swap.to_address,
-            current_swap.token_out_address,
-            other_swaps,
-        )
-
-        if next_swap is None:
-            return None
-
-        swap_path.append(next_swap)
-        current_swap = next_swap
-
-        if (
-            current_swap.to_address == start_swap.from_address
-            and current_swap.token_out_address == start_swap.token_in_address
-        ):
-
-            start_amount = start_swap.token_in_amount
-            end_amount = current_swap.token_out_amount
             profit_amount = end_amount - start_amount
 
-            return Arbitrage(
+            arb = Arbitrage(
-                swaps=swap_path,
+                swaps=route,
-                block_number=start_swap.block_number,
+                block_number=route[0].block_number,
-                transaction_hash=start_swap.transaction_hash,
+                transaction_hash=route[0].transaction_hash,
-                account_address=start_swap.from_address,
+                account_address=route[0].from_address,
-                profit_token_address=start_swap.token_in_address,
+                profit_token_address=route[0].token_in_address,
                 start_amount=start_amount,
                 end_amount=end_amount,
                 profit_amount=profit_amount,
             )
-
+            all_arbitrages.append(arb)
-    return None
+    if len(all_arbitrages) == 1:
+        return all_arbitrages
+    else:
+        return [
+            arb
+            for arb in all_arbitrages
+            if (arb.swaps[0].trace_address < arb.swaps[-1].trace_address)
+        ]
 
 
-def _get_swap_from_address(
+def _get_all_start_end_swaps(swaps: List[Swap]) -> List[Tuple[Swap, Swap]]:
-    address: str, token_address: str, swaps: List[Swap]
+    """
-) -> Optional[Swap]:
+    Gets the set of all possible opening and closing swap pairs in an arbitrage via
-    for swap in swaps:
+    - swap[start].token_in == swap[end].token_out
-        if swap.pool_address == address and swap.token_in_address == token_address:
+    - swap[start].from_address == swap[end].to_address
-            return swap
+    - not swap[start].from_address in all_pool_addresses
+    - not swap[end].to_address in all_pool_addresses
+    """
+    pool_addrs = [swap.contract_address for swap in swaps]
+    valid_start_ends: List[Tuple[Swap, Swap]] = []
+    for potential_start_swap in swaps:
+        for potential_end_swap in swaps:
+            if (
+                potential_start_swap.token_in_address
+                == potential_end_swap.token_out_address
+                and potential_start_swap.from_address == potential_end_swap.to_address
+                and not potential_start_swap.from_address in pool_addrs
+            ):
+                valid_start_ends.append((potential_start_swap, potential_end_swap))
+    return valid_start_ends
 
-    return None
+
+def _get_all_routes(
+    start_swap: Swap, end_swap: Swap, other_swaps: List[Swap]
+) -> List[List[Swap]]:
+    """
+    Returns all routes (List[Swap]) from start to finish between a start_swap and an end_swap only accounting for token_address_in and token_address_out.
+    """
+    # If the path is complete, return
+    if start_swap.token_out_address == end_swap.token_in_address:
+        return [[start_swap, end_swap]]
+    elif len(other_swaps) == 0:
+        return []
+
+    # Collect all potential next steps, check if valid, recursively find routes from next_step to end_swap
+    routes: List[List[Swap]] = []
+    for potential_next_swap in other_swaps:
+        if start_swap.token_out_address == potential_next_swap.token_in_address and (
+            start_swap.contract_address == potential_next_swap.from_address
+            or start_swap.to_address == potential_next_swap.contract_address
+            or start_swap.to_address == potential_next_swap.from_address
+        ):
+            remaining_swaps = [
+                swap for swap in other_swaps if swap != potential_next_swap
+            ]
+            next_swap_routes = _get_all_routes(
+                potential_next_swap, end_swap, remaining_swaps
+            )
+            if len(next_swap_routes) > 0:
+                for next_swap_route in next_swap_routes:
+                    next_swap_route.insert(0, start_swap)
+                    routes.append(next_swap_route)
+    return routes

mev_inspect/block.py

@@ -1,4 +1,5 @@
-from pathlib import Path
+import asyncio
+import logging
 from typing import List, Optional
 import json
 import asyncio
@@ -8,18 +9,25 @@ from sqlalchemy import orm
 from web3 import Web3
 
 from mev_inspect.fees import fetch_base_fee_per_gas
-from mev_inspect.schemas import Block, Trace, TraceType
+from mev_inspect.schemas.blocks import Block
 from mev_inspect.schemas.receipts import Receipt
+from mev_inspect.schemas.traces import Trace, TraceType
+from mev_inspect.utils import hex_to_int
 
 
-cache_directory = "./cache"
+logger = logging.getLogger(__name__)
 
 
-def get_latest_block_number(w3: Web3) -> int:
+async def get_latest_block_number(base_provider) -> int:
-    return int(w3.eth.get_block("latest")["number"])
+    latest_block = await base_provider.make_request(
+        "eth_getBlockByNumber",
+        ["latest", False],
+    )
+
+    return hex_to_int(latest_block["result"]["number"])
 
 
-def create_from_block_number(
+async def create_from_block_number(
     base_provider,
     w3: Web3,
     geth: bool,
@@ -32,30 +40,35 @@ def create_from_block_number(
         block = _find_block(trace_db_session, block_number)
 
     if block is None:
-        return _fetch_block(w3, base_provider, geth, block_number)
+        block = await _fetch_block(w3, base_provider, block_number)
+        return block
     else:
         return block
 
 
-def _fetch_block(
+async def _fetch_block(w3, base_provider, geth, block_number: int, retries: int = 0) -> Block:
-    w3,
-    base_provider,
-    geth,
-    block_number: int,
-) -> Block:
-    block_json = w3.eth.get_block(block_number)
-
     if not geth:
-        receipts_json = base_provider.make_request(
+        block_json, receipts_json, traces_json, base_fee_per_gas = await asyncio.gather(
-            "eth_getBlockReceipts", [block_number]
+            w3.eth.get_block(block_number),
+            base_provider.make_request("eth_getBlockReceipts", [block_number]),
+            base_provider.make_request("trace_block", [block_number]),
+            fetch_base_fee_per_gas(w3, block_number),
         )
-        traces_json = w3.parity.trace_block(block_number)
 
-        receipts: List[Receipt] = [
+        try:
-            Receipt(**receipt) for receipt in receipts_json["result"]
+            receipts: List[Receipt] = [
-        ]
+                Receipt(**receipt) for receipt in receipts_json["result"]
-        traces = [Trace(**trace_json) for trace_json in traces_json]
+            ]
-        base_fee_per_gas = fetch_base_fee_per_gas(w3, block_number)
+            traces = [Trace(**trace_json) for trace_json in traces_json["result"]]
+        except KeyError as e:
+            logger.warning(
+                f"Failed to create objects from block: {block_number}: {e}, retrying: {retries + 1} / 3"
+            )
+            if retries < 3:
+                await asyncio.sleep(5)
+                return await _fetch_block(w3, base_provider, block_number, retries)
+            else:
+                raise
     else:
         traces = geth_get_tx_traces_parity_format(base_provider, block_json)
         geth_tx_receipts = geth_get_tx_receipts(
@@ -63,25 +76,32 @@ def _fetch_block(
         )
         receipts = geth_receipts_translator(block_json, geth_tx_receipts)
         base_fee_per_gas = 0
-
+        
-    return Block(
+        return Block(
-        block_number=block_number,
+            block_number=block_number,
-        miner=block_json["miner"],
+            block_timestamp=block_json["timestamp"],
-        base_fee_per_gas=base_fee_per_gas,
+            miner=block_json["miner"],
-        traces=traces,
+            base_fee_per_gas=base_fee_per_gas,
-        receipts=receipts,
+            traces=traces,
-    )
+            receipts=receipts,
-
+        )
+ 
 
 def _find_block(
     trace_db_session: orm.Session,
     block_number: int,
 ) -> Optional[Block]:
+    block_timestamp = _find_block_timestamp(trace_db_session, block_number)
     traces = _find_traces(trace_db_session, block_number)
     receipts = _find_receipts(trace_db_session, block_number)
     base_fee_per_gas = _find_base_fee(trace_db_session, block_number)
 
-    if traces is None or receipts is None or base_fee_per_gas is None:
+    if (
+        block_timestamp is None
+        or traces is None
+        or receipts is None
+        or base_fee_per_gas is None
+    ):
         return None
 
     miner_address = _get_miner_address_from_traces(traces)
@@ -91,12 +111,28 @@ def _find_block(
 
     return Block(
         block_number=block_number,
+        block_timestamp=block_timestamp,
         miner=miner_address,
         base_fee_per_gas=base_fee_per_gas,
         traces=traces,
         receipts=receipts,
     )
 
+def _find_block_timestamp(
+    trace_db_session: orm.Session,
+    block_number: int,
+) -> Optional[int]:
+    result = trace_db_session.execute(
+        "SELECT block_timestamp FROM block_timestamps WHERE block_number = :block_number",
+        params={"block_number": block_number},
+    ).one_or_none()
+
+    if result is None:
+        return None
+    else:
+        (block_timestamp,) = result
+        return block_timestamp
+
 
 def _find_traces(
     trace_db_session: orm.Session,
@@ -167,21 +203,6 @@ def get_transaction_hashes(calls: List[Trace]) -> List[str]:
 
     return result
 
-
-def cache_block(cache_path: Path, block: Block):
-    write_mode = "w" if cache_path.is_file() else "x"
-
-    cache_path.parent.mkdir(parents=True, exist_ok=True)
-
-    with open(cache_path, mode=write_mode) as cache_file:
-        cache_file.write(block.json())
-
-
-def _get_cache_path(block_number: int) -> Path:
-    cache_directory_path = Path(cache_directory)
-    return cache_directory_path / f"{block_number}.json"
-
-
 # Geth specific additions
 
 

mev_inspect/classifiers/helpers.py

@@ -0,0 +1,86 @@
+from typing import Optional, List, Sequence
+
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.transfers import Transfer, ETH_TOKEN_ADDRESS
+
+from mev_inspect.schemas.traces import DecodedCallTrace, ClassifiedTrace
+
+
+def create_swap_from_transfers(
+    trace: DecodedCallTrace,
+    recipient_address: str,
+    prior_transfers: List[Transfer],
+    child_transfers: List[Transfer],
+) -> Optional[Swap]:
+    pool_address = trace.to_address
+
+    transfers_to_pool = []
+
+    if trace.value is not None and trace.value > 0:
+        transfers_to_pool = [_build_eth_transfer(trace)]
+
+    if len(transfers_to_pool) == 0:
+        transfers_to_pool = _filter_transfers(prior_transfers, to_address=pool_address)
+
+    if len(transfers_to_pool) == 0:
+        transfers_to_pool = _filter_transfers(child_transfers, to_address=pool_address)
+
+    if len(transfers_to_pool) == 0:
+        return None
+
+    transfers_from_pool_to_recipient = _filter_transfers(
+        child_transfers, to_address=recipient_address, from_address=pool_address
+    )
+
+    if len(transfers_from_pool_to_recipient) != 1:
+        return None
+
+    transfer_in = transfers_to_pool[-1]
+    transfer_out = transfers_from_pool_to_recipient[0]
+
+    return Swap(
+        abi_name=trace.abi_name,
+        transaction_hash=trace.transaction_hash,
+        block_number=trace.block_number,
+        trace_address=trace.trace_address,
+        contract_address=pool_address,
+        protocol=trace.protocol,
+        from_address=transfer_in.from_address,
+        to_address=transfer_out.to_address,
+        token_in_address=transfer_in.token_address,
+        token_in_amount=transfer_in.amount,
+        token_out_address=transfer_out.token_address,
+        token_out_amount=transfer_out.amount,
+        error=trace.error,
+    )
+
+
+def _build_eth_transfer(trace: ClassifiedTrace) -> Transfer:
+    return Transfer(
+        block_number=trace.block_number,
+        transaction_hash=trace.transaction_hash,
+        trace_address=trace.trace_address,
+        amount=trace.value,
+        to_address=trace.to_address,
+        from_address=trace.from_address,
+        token_address=ETH_TOKEN_ADDRESS,
+    )
+
+
+def _filter_transfers(
+    transfers: Sequence[Transfer],
+    to_address: Optional[str] = None,
+    from_address: Optional[str] = None,
+) -> List[Transfer]:
+    filtered_transfers = []
+
+    for transfer in transfers:
+        if to_address is not None and transfer.to_address != to_address:
+            continue
+
+        if from_address is not None and transfer.from_address != from_address:
+            continue
+
+        filtered_transfers.append(transfer)
+
+    return filtered_transfers

mev_inspect/classifiers/specs/__init__.py

@@ -1,6 +1,6 @@
 from typing import Dict, Optional, Tuple, Type
 
-from mev_inspect.schemas.classified_traces import DecodedCallTrace, Protocol
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
 from mev_inspect.schemas.classifiers import ClassifierSpec, Classifier
 
 from .aave import AAVE_CLASSIFIER_SPECS

mev_inspect/classifiers/specs/aave.py

@@ -1,13 +1,10 @@
-from mev_inspect.schemas.classified_traces import (
-    Protocol,
-)
-
 from mev_inspect.schemas.classifiers import (
     ClassifierSpec,
     DecodedCallTrace,
     TransferClassifier,
     LiquidationClassifier,
 )
+from mev_inspect.schemas.traces import Protocol
 from mev_inspect.schemas.transfers import Transfer
 
 

mev_inspect/classifiers/specs/balancer.py

@@ -1,4 +1,7 @@
-from mev_inspect.schemas.classified_traces import (
+from typing import Optional, List
+from mev_inspect.schemas.transfers import Transfer
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import (
     DecodedCallTrace,
     Protocol,
 )
@@ -6,15 +9,25 @@ from mev_inspect.schemas.classifiers import (
     ClassifierSpec,
     SwapClassifier,
 )
-
+from mev_inspect.classifiers.helpers import create_swap_from_transfers
 
 BALANCER_V1_POOL_ABI_NAME = "BPool"
 
 
 class BalancerSwapClassifier(SwapClassifier):
     @staticmethod
-    def get_swap_recipient(trace: DecodedCallTrace) -> str:
+    def parse_swap(
-        return trace.from_address
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
+
+        recipient_address = trace.from_address
+
+        swap = create_swap_from_transfers(
+            trace, recipient_address, prior_transfers, child_transfers
+        )
+        return swap
 
 
 BALANCER_V1_SPECS = [

mev_inspect/classifiers/specs/compound.py

@@ -1,4 +1,4 @@
-from mev_inspect.schemas.classified_traces import (
+from mev_inspect.schemas.traces import (
     Protocol,
 )
 from mev_inspect.schemas.classifiers import (

mev_inspect/classifiers/specs/curve.py

@@ -1,18 +1,32 @@
-from mev_inspect.schemas.classified_traces import (
+from typing import Optional, List
+from mev_inspect.schemas.transfers import Transfer
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import (
     Protocol,
+    DecodedCallTrace,
 )
 
 from mev_inspect.schemas.classifiers import (
     ClassifierSpec,
-    DecodedCallTrace,
     SwapClassifier,
 )
+from mev_inspect.classifiers.helpers import create_swap_from_transfers
 
 
 class CurveSwapClassifier(SwapClassifier):
     @staticmethod
-    def get_swap_recipient(trace: DecodedCallTrace) -> str:
+    def parse_swap(
-        return trace.from_address
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
+
+        recipient_address = trace.from_address
+
+        swap = create_swap_from_transfers(
+            trace, recipient_address, prior_transfers, child_transfers
+        )
+        return swap
 
 
 CURVE_BASE_POOLS = [

mev_inspect/classifiers/specs/erc20.py

@@ -1,4 +1,4 @@
-from mev_inspect.schemas.classified_traces import DecodedCallTrace
+from mev_inspect.schemas.traces import DecodedCallTrace
 from mev_inspect.schemas.classifiers import (
     ClassifierSpec,
     TransferClassifier,

mev_inspect/classifiers/specs/uniswap.py

@@ -1,4 +1,7 @@
-from mev_inspect.schemas.classified_traces import (
+from typing import Optional, List
+from mev_inspect.schemas.transfers import Transfer
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import (
     DecodedCallTrace,
     Protocol,
 )
@@ -6,6 +9,7 @@ from mev_inspect.schemas.classifiers import (
     ClassifierSpec,
     SwapClassifier,
 )
+from mev_inspect.classifiers.helpers import create_swap_from_transfers
 
 
 UNISWAP_V2_PAIR_ABI_NAME = "UniswapV2Pair"
@@ -14,20 +18,34 @@ UNISWAP_V3_POOL_ABI_NAME = "UniswapV3Pool"
 
 class UniswapV3SwapClassifier(SwapClassifier):
     @staticmethod
-    def get_swap_recipient(trace: DecodedCallTrace) -> str:
+    def parse_swap(
-        if trace.inputs is not None and "recipient" in trace.inputs:
+        trace: DecodedCallTrace,
-            return trace.inputs["recipient"]
+        prior_transfers: List[Transfer],
-        else:
+        child_transfers: List[Transfer],
-            return trace.from_address
+    ) -> Optional[Swap]:
+
+        recipient_address = trace.inputs.get("recipient", trace.from_address)
+
+        swap = create_swap_from_transfers(
+            trace, recipient_address, prior_transfers, child_transfers
+        )
+        return swap
 
 
 class UniswapV2SwapClassifier(SwapClassifier):
     @staticmethod
-    def get_swap_recipient(trace: DecodedCallTrace) -> str:
+    def parse_swap(
-        if trace.inputs is not None and "to" in trace.inputs:
+        trace: DecodedCallTrace,
-            return trace.inputs["to"]
+        prior_transfers: List[Transfer],
-        else:
+        child_transfers: List[Transfer],
-            return trace.from_address
+    ) -> Optional[Swap]:
+
+        recipient_address = trace.inputs.get("to", trace.from_address)
+
+        swap = create_swap_from_transfers(
+            trace, recipient_address, prior_transfers, child_transfers
+        )
+        return swap
 
 
 UNISWAP_V3_CONTRACT_SPECS = [
@@ -127,7 +145,7 @@ UNISWAPPY_V2_PAIR_SPEC = ClassifierSpec(
     },
 )
 
-UNISWAP_CLASSIFIER_SPECS = [
+UNISWAP_CLASSIFIER_SPECS: List = [
     *UNISWAP_V3_CONTRACT_SPECS,
     *UNISWAPPY_V2_CONTRACT_SPECS,
     *UNISWAP_V3_GENERAL_SPECS,

mev_inspect/classifiers/specs/weth.py

@@ -1,4 +1,4 @@
-from mev_inspect.schemas.classified_traces import (
+from mev_inspect.schemas.traces import (
     Protocol,
 )
 from mev_inspect.schemas.classifiers import (

mev_inspect/classifiers/specs/zero_ex.py

@@ -1,11 +1,10 @@
-from mev_inspect.schemas.classified_traces import (
+from mev_inspect.schemas.traces import (
     Protocol,
 )
 from mev_inspect.schemas.classifiers import (
     ClassifierSpec,
 )
 
-
 ZEROX_CONTRACT_SPECS = [
     ClassifierSpec(
         abi_name="exchangeProxy",

mev_inspect/classifiers/trace.py

@@ -2,13 +2,14 @@ from typing import Dict, List, Optional
 
 from mev_inspect.abi import get_abi
 from mev_inspect.decode import ABIDecoder
-from mev_inspect.schemas.blocks import CallAction, CallResult, Trace, TraceType
+from mev_inspect.schemas.blocks import CallAction, CallResult
-from mev_inspect.schemas.classified_traces import (
+from mev_inspect.schemas.traces import (
     Classification,
     ClassifiedTrace,
     CallTrace,
     DecodedCallTrace,
 )
+from mev_inspect.schemas.traces import Trace, TraceType
 
 from .specs import ALL_CLASSIFIER_SPECS
 

mev_inspect/compound_liquidations.py

@@ -2,7 +2,7 @@ from typing import Dict, List, Optional
 from web3 import Web3
 
 from mev_inspect.traces import get_child_traces
-from mev_inspect.schemas.classified_traces import (
+from mev_inspect.schemas.traces import (
     ClassifiedTrace,
     Classification,
     Protocol,

mev_inspect/concurrency.py

@@ -0,0 +1,22 @@
+import asyncio
+import signal
+from functools import wraps
+
+
+def coro(f):
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        loop = asyncio.get_event_loop()
+
+        def cancel_task_callback():
+            for task in asyncio.all_tasks():
+                task.cancel()
+
+        for sig in (signal.SIGINT, signal.SIGTERM):
+            loop.add_signal_handler(sig, cancel_task_callback)
+        try:
+            loop.run_until_complete(f(*args, **kwargs))
+        finally:
+            loop.run_until_complete(loop.shutdown_asyncgens())
+
+    return wrapper

mev_inspect/crud/blocks.py

@@ -0,0 +1,26 @@
+from mev_inspect.schemas.blocks import Block
+
+
+def delete_block(
+    db_session,
+    block_number: int,
+) -> None:
+    db_session.execute(
+        "DELETE FROM blocks WHERE block_number = :block_number",
+        params={"block_number": block_number},
+    )
+    db_session.commit()
+
+
+def write_block(
+    db_session,
+    block: Block,
+) -> None:
+    db_session.execute(
+        "INSERT INTO blocks (block_number, block_timestamp) VALUES (:block_number, :block_timestamp)",
+        params={
+            "block_number": block.block_number,
+            "block_timestamp": block.block_timestamp,
+        },
+    )
+    db_session.commit()

mev_inspect/crud/traces.py

@@ -1,8 +1,8 @@
 import json
 from typing import List
 
-from mev_inspect.models.classified_traces import ClassifiedTraceModel
+from mev_inspect.models.traces import ClassifiedTraceModel
-from mev_inspect.schemas.classified_traces import ClassifiedTrace
+from mev_inspect.schemas.traces import ClassifiedTrace
 
 
 def delete_classified_traces_for_block(

mev_inspect/fees.py

@@ -1,9 +1,10 @@
 from web3 import Web3
 
 
-def fetch_base_fee_per_gas(w3: Web3, block_number: int) -> int:
+async def fetch_base_fee_per_gas(w3: Web3, block_number: int) -> int:
-    base_fees = w3.eth.fee_history(1, block_number)["baseFeePerGas"]
+    base_fees = await w3.eth.fee_history(1, block_number)
-    if len(base_fees) == 0:
+    base_fees_per_gas = base_fees["baseFeePerGas"]
+    if len(base_fees_per_gas) == 0:
         raise RuntimeError("Unexpected error - no fees returned")
 
-    return base_fees[0]
+    return base_fees_per_gas[0]

mev_inspect/inspect_block.py

@@ -11,7 +11,11 @@ from mev_inspect.crud.arbitrages import (
     delete_arbitrages_for_block,
     write_arbitrages,
 )
-from mev_inspect.crud.classified_traces import (
+from mev_inspect.crud.blocks import (
+    delete_block,
+    write_block,
+)
+from mev_inspect.crud.traces import (
     delete_classified_traces_for_block,
     write_classified_traces,
 )
@@ -35,7 +39,7 @@ from mev_inspect.liquidations import get_liquidations
 logger = logging.getLogger(__name__)
 
 
-def inspect_block(
+async def inspect_block(
     inspect_db_session: orm.Session,
     base_provider,
     w3: Web3,
@@ -45,7 +49,7 @@ def inspect_block(
     trace_db_session: Optional[orm.Session],
     should_write_classified_traces: bool = True,
 ):
-    block = create_from_block_number(
+    block = await create_from_block_number(
         base_provider,
         w3,
         geth,
@@ -53,40 +57,45 @@ def inspect_block(
         trace_db_session,
     )
 
-    logger.info(f"Total traces: {len(block.traces)}")
+    logger.info(f"Block: {block_number} -- Total traces: {len(block.traces)}")
+
+    delete_block(inspect_db_session, block_number)
+    write_block(inspect_db_session, block)
 
     total_transactions = len(
         set(t.transaction_hash for t in block.traces if t.transaction_hash is not None)
     )
-    logger.info(f"Total transactions: {total_transactions}")
+    logger.info(f"Block: {block_number} -- Total transactions: {total_transactions}")
 
-    classified_traces = trace_clasifier.classify(block.traces)
+    classified_traces = trace_classifier.classify(block.traces)
-    logger.info(f"Returned {len(classified_traces)} classified traces")
+    logger.info(
+        f"Block: {block_number} -- Returned {len(classified_traces)} classified traces"
+    )
 
     if should_write_classified_traces:
         delete_classified_traces_for_block(inspect_db_session, block_number)
         write_classified_traces(inspect_db_session, classified_traces)
 
     transfers = get_transfers(classified_traces)
-    logger.info(f"Found {len(transfers)} transfers")
+    logger.info(f"Block: {block_number} -- Found {len(transfers)} transfers")
 
     delete_transfers_for_block(inspect_db_session, block_number)
     write_transfers(inspect_db_session, transfers)
 
     swaps = get_swaps(classified_traces)
-    logger.info(f"Found {len(swaps)} swaps")
+    logger.info(f"Block: {block_number} -- Found {len(swaps)} swaps")
 
     delete_swaps_for_block(inspect_db_session, block_number)
     write_swaps(inspect_db_session, swaps)
 
     arbitrages = get_arbitrages(swaps)
-    logger.info(f"Found {len(arbitrages)} arbitrages")
+    logger.info(f"Block: {block_number} -- Found {len(arbitrages)} arbitrages")
 
     delete_arbitrages_for_block(inspect_db_session, block_number)
     write_arbitrages(inspect_db_session, arbitrages)
 
     liquidations = get_liquidations(classified_traces)
-    logger.info(f"Found {len(liquidations)} liquidations")
+    logger.info(f"Block: {block_number} -- Found {len(liquidations)} liquidations")
 
     delete_liquidations_for_block(inspect_db_session, block_number)
     write_liquidations(inspect_db_session, liquidations)

mev_inspect/inspector.py

@@ -0,0 +1,79 @@
+import asyncio
+import logging
+import traceback
+from asyncio import CancelledError
+from typing import Optional
+
+from sqlalchemy import orm
+from web3 import Web3
+from web3.eth import AsyncEth
+
+from mev_inspect.block import create_from_block_number
+from mev_inspect.classifiers.trace import TraceClassifier
+from mev_inspect.inspect_block import inspect_block
+from mev_inspect.provider import get_base_provider
+
+logger = logging.getLogger(__name__)
+
+
+class MEVInspector:
+    def __init__(
+        self,
+        rpc: str,
+        inspect_db_session: orm.Session,
+        trace_db_session: Optional[orm.Session],
+        max_concurrency: int = 1,
+        request_timeout: int = 300,
+    ):
+        self.inspect_db_session = inspect_db_session
+        self.trace_db_session = trace_db_session
+        self.base_provider = get_base_provider(rpc, request_timeout=request_timeout)
+        self.w3 = Web3(self.base_provider, modules={"eth": (AsyncEth,)}, middlewares=[])
+        self.trace_classifier = TraceClassifier()
+        self.max_concurrency = asyncio.Semaphore(max_concurrency)
+
+    async def create_from_block(self, block_number: int):
+        return await create_from_block_number(
+            base_provider=self.base_provider,
+            w3=self.w3,
+            block_number=block_number,
+            trace_db_session=self.trace_db_session,
+        )
+
+    async def inspect_single_block(self, block: int):
+        return await inspect_block(
+            self.inspect_db_session,
+            self.base_provider,
+            self.w3,
+            self.trace_classifier,
+            block,
+            trace_db_session=self.trace_db_session,
+        )
+
+    async def inspect_many_blocks(self, after_block: int, before_block: int):
+        tasks = []
+        for block_number in range(after_block, before_block):
+            tasks.append(
+                asyncio.ensure_future(
+                    self.safe_inspect_block(block_number=block_number)
+                )
+            )
+        logger.info(f"Gathered {len(tasks)} blocks to inspect")
+        try:
+            await asyncio.gather(*tasks)
+        except CancelledError:
+            logger.info("Requested to exit, cleaning up...")
+        except Exception as e:
+            logger.error(f"Existed due to {type(e)}")
+            traceback.print_exc()
+
+    async def safe_inspect_block(self, block_number: int):
+        async with self.max_concurrency:
+            return await inspect_block(
+                self.inspect_db_session,
+                self.base_provider,
+                self.w3,
+                self.trace_classifier,
+                block_number,
+                trace_db_session=self.trace_db_session,
+            )

mev_inspect/liquidations.py

@@ -1,7 +1,7 @@
 from typing import List
 
 from mev_inspect.aave_liquidations import get_aave_liquidations
-from mev_inspect.schemas.classified_traces import (
+from mev_inspect.schemas.traces import (
     ClassifiedTrace,
     Classification,
 )

mev_inspect/miner_payments.py

@@ -1,6 +1,6 @@
 from typing import List
 
-from mev_inspect.schemas.classified_traces import ClassifiedTrace
+from mev_inspect.schemas.traces import ClassifiedTrace
 from mev_inspect.schemas.miner_payments import MinerPayment
 from mev_inspect.schemas.receipts import Receipt
 from mev_inspect.traces import get_traces_by_transaction_hash

mev_inspect/models/swaps.py

@@ -11,7 +11,7 @@ class SwapModel(Base):
     block_number = Column(Numeric, nullable=False)
     trace_address = Column(ARRAY(Integer), nullable=False)
     protocol = Column(String, nullable=True)
-    pool_address = Column(String, nullable=False)
+    contract_address = Column(String, nullable=False)
     from_address = Column(String, nullable=False)
     to_address = Column(String, nullable=False)
     token_in_address = Column(String, nullable=False)

mev_inspect/provider.py

@@ -1,14 +1,9 @@
-from web3 import Web3
+from web3 import Web3, AsyncHTTPProvider
 
 from mev_inspect.retry import http_retry_with_backoff_request_middleware
 
 
-def get_base_provider(rpc: str) -> Web3.HTTPProvider:
+def get_base_provider(rpc: str, request_timeout: int = 500) -> Web3.AsyncHTTPProvider:
-    base_provider = Web3.HTTPProvider(rpc)
+    base_provider = AsyncHTTPProvider(rpc, request_kwargs={"timeout": request_timeout})
-    base_provider.middlewares.remove("http_retry_request")
+    base_provider.middlewares += (http_retry_with_backoff_request_middleware,)
-    base_provider.middlewares.add(
-        http_retry_with_backoff_request_middleware,
-        "http_retry_with_backoff",
-    )
-
     return base_provider

mev_inspect/retry.py

@@ -1,11 +1,21 @@
-import time
+import asyncio
+import logging
+import random
 from typing import (
     Any,
     Callable,
     Collection,
     Type,
+    Coroutine,
 )
+from asyncio.exceptions import TimeoutError
 
+from aiohttp.client_exceptions import (
+    ClientOSError,
+    ServerDisconnectedError,
+    ServerTimeoutError,
+    ClientResponseError,
+)
 from requests.exceptions import (
     ConnectionError,
     HTTPError,
@@ -20,40 +30,61 @@ from web3.types import (
 )
 
 
-def exception_retry_with_backoff_middleware(
+request_exceptions = (ConnectionError, HTTPError, Timeout, TooManyRedirects)
-    make_request: Callable[[RPCEndpoint, Any], RPCResponse],
+aiohttp_exceptions = (
+    ClientOSError,
+    ServerDisconnectedError,
+    ServerTimeoutError,
+    ClientResponseError,
+)
+
+logger = logging.getLogger(__name__)
+
+
+async def exception_retry_with_backoff_middleware(
+    make_request: Callable[[RPCEndpoint, Any], Any],
     web3: Web3,  # pylint: disable=unused-argument
     errors: Collection[Type[BaseException]],
     retries: int = 5,
     backoff_time_seconds: float = 0.1,
-) -> Callable[[RPCEndpoint, Any], RPCResponse]:
+) -> Callable[[RPCEndpoint, Any], Coroutine[Any, Any, RPCResponse]]:
     """
     Creates middleware that retries failed HTTP requests. Is a default
     middleware for HTTPProvider.
     """
 
-    def middleware(method: RPCEndpoint, params: Any) -> RPCResponse:
+    async def middleware(method: RPCEndpoint, params: Any) -> RPCResponse:
+
         if check_if_retry_on_failure(method):
             for i in range(retries):
                 try:
-                    return make_request(method, params)
+                    return await make_request(method, params)
                 # https://github.com/python/mypy/issues/5349
                 except errors:  # type: ignore
+                    logger.error(
+                        f"Request for method {method}, block: {int(params[0], 16)}, retrying: {i}/{retries}"
+                    )
                     if i < retries - 1:
-                        time.sleep(backoff_time_seconds)
+                        backoff_time = backoff_time_seconds * (
+                            random.uniform(5, 10) ** i
+                        )
+                        await asyncio.sleep(backoff_time)
                         continue
+
                     else:
                         raise
             return None
         else:
-            return make_request(method, params)
+            return await make_request(method, params)
 
     return middleware
 
 
-def http_retry_with_backoff_request_middleware(
+async def http_retry_with_backoff_request_middleware(
     make_request: Callable[[RPCEndpoint, Any], Any], web3: Web3
-) -> Callable[[RPCEndpoint, Any], Any]:
+) -> Callable[[RPCEndpoint, Any], Coroutine[Any, Any, RPCResponse]]:
-    return exception_retry_with_backoff_middleware(
+    return await exception_retry_with_backoff_middleware(
-        make_request, web3, (ConnectionError, HTTPError, Timeout, TooManyRedirects)
+        make_request,
+        web3,
+        (request_exceptions + aiohttp_exceptions + (TimeoutError,)),
     )

mev_inspect/schemas/__init__.py

@@ -1,2 +0,0 @@
-from .abi import ABI
-from .blocks import Block, Trace, TraceType

mev_inspect/schemas/blocks.py

@@ -1,11 +1,11 @@
-from enum import Enum
+from typing import List
-from typing import List, Optional
 
 from pydantic import validator
 
 from mev_inspect.utils import hex_to_int
 
 from .receipts import Receipt
+from .traces import Trace
 from .utils import CamelModel, Web3Model
 
 
@@ -36,29 +36,9 @@ class CallAction(Web3Model):
         fields = {"from_": "from"}
 
 
-class TraceType(Enum):
-    call = "call"
-    create = "create"
-    delegate_call = "delegateCall"
-    reward = "reward"
-    suicide = "suicide"
-
-
-class Trace(CamelModel):
-    action: dict
-    block_hash: str
-    block_number: int
-    result: Optional[dict]
-    subtraces: int
-    trace_address: List[int]
-    transaction_hash: Optional[str]
-    transaction_position: Optional[int]
-    type: TraceType
-    error: Optional[str]
-
-
 class Block(Web3Model):
     block_number: int
+    block_timestamp: int
     miner: str
     base_fee_per_gas: int
     traces: List[Trace]

mev_inspect/schemas/classifiers.py

@@ -3,8 +3,9 @@ from typing import Dict, List, Optional, Type
 
 from pydantic import BaseModel
 
-from .classified_traces import Classification, DecodedCallTrace, Protocol
+from .traces import Classification, DecodedCallTrace, Protocol
 from .transfers import Transfer
+from .swaps import Swap
 
 
 class Classifier(ABC):
@@ -32,7 +33,11 @@ class SwapClassifier(Classifier):
 
     @staticmethod
     @abstractmethod
-    def get_swap_recipient(trace: DecodedCallTrace) -> str:
+    def parse_swap(
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
         raise NotImplementedError()
 
 

mev_inspect/schemas/liquidations.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 from pydantic import BaseModel
-from mev_inspect.schemas.classified_traces import Protocol
+from mev_inspect.schemas.traces import Protocol
 
 
 class Liquidation(BaseModel):

mev_inspect/schemas/swaps.py

@@ -2,7 +2,7 @@ from typing import List, Optional
 
 from pydantic import BaseModel
 
-from mev_inspect.schemas.classified_traces import Protocol
+from mev_inspect.schemas.traces import Protocol
 
 
 class Swap(BaseModel):
@@ -10,7 +10,7 @@ class Swap(BaseModel):
     transaction_hash: str
     block_number: int
     trace_address: List[int]
-    pool_address: str
+    contract_address: str
     from_address: str
     to_address: str
     token_in_address: str

mev_inspect/schemas/traces.py

@@ -1,7 +1,28 @@
 from enum import Enum
 from typing import Any, Dict, List, Optional
 
-from .blocks import Trace
+from .utils import CamelModel
+
+
+class TraceType(Enum):
+    call = "call"
+    create = "create"
+    delegate_call = "delegateCall"
+    reward = "reward"
+    suicide = "suicide"
+
+
+class Trace(CamelModel):
+    action: dict
+    block_hash: str
+    block_number: int
+    result: Optional[dict]
+    subtraces: int
+    trace_address: List[int]
+    transaction_hash: Optional[str]
+    transaction_position: Optional[int]
+    type: TraceType
+    error: Optional[str]
 
 
 class Classification(Enum):
@@ -26,16 +47,13 @@ class Protocol(Enum):
 
 
 class ClassifiedTrace(Trace):
-    transaction_hash: str
-    block_number: int
-    trace_address: List[int]
     classification: Classification
-    error: Optional[str]
     to_address: Optional[str]
     from_address: Optional[str]
     gas: Optional[int]
     value: Optional[int]
     gas_used: Optional[int]
+    transaction_hash: str
     protocol: Optional[Protocol]
     function_name: Optional[str]
     function_signature: Optional[str]

mev_inspect/schemas/transfers.py

@@ -3,7 +3,7 @@ from typing import List
 from pydantic import BaseModel
 
 
-ETH_TOKEN_ADDRESS = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE"
+ETH_TOKEN_ADDRESS = "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"
 
 
 class Transfer(BaseModel):

mev_inspect/swaps.py

@@ -1,7 +1,7 @@
 from typing import List, Optional
 
 from mev_inspect.classifiers.specs import get_classifier
-from mev_inspect.schemas.classified_traces import (
+from mev_inspect.schemas.traces import (
     ClassifiedTrace,
     Classification,
     DecodedCallTrace,
@@ -11,10 +11,8 @@ from mev_inspect.schemas.swaps import Swap
 from mev_inspect.schemas.transfers import Transfer
 from mev_inspect.traces import get_traces_by_transaction_hash
 from mev_inspect.transfers import (
-    build_eth_transfer,
     get_child_transfers,
     get_transfer,
-    filter_transfers,
     remove_child_transfers_of_transfers,
 )
 
@@ -67,56 +65,8 @@ def _parse_swap(
     prior_transfers: List[Transfer],
     child_transfers: List[Transfer],
 ) -> Optional[Swap]:
-    pool_address = trace.to_address
-    recipient_address = _get_recipient_address(trace)
 
-    if recipient_address is None:
-        return None
-
-    transfers_to_pool = []
-
-    if trace.value is not None and trace.value > 0:
-        transfers_to_pool = [build_eth_transfer(trace)]
-
-    if len(transfers_to_pool) == 0:
-        transfers_to_pool = filter_transfers(prior_transfers, to_address=pool_address)
-
-    if len(transfers_to_pool) == 0:
-        transfers_to_pool = filter_transfers(child_transfers, to_address=pool_address)
-
-    if len(transfers_to_pool) == 0:
-        return None
-
-    transfers_from_pool_to_recipient = filter_transfers(
-        child_transfers, to_address=recipient_address, from_address=pool_address
-    )
-
-    if len(transfers_from_pool_to_recipient) != 1:
-        return None
-
-    transfer_in = transfers_to_pool[-1]
-    transfer_out = transfers_from_pool_to_recipient[0]
-
-    return Swap(
-        abi_name=trace.abi_name,
-        transaction_hash=trace.transaction_hash,
-        block_number=trace.block_number,
-        trace_address=trace.trace_address,
-        pool_address=pool_address,
-        protocol=trace.protocol,
-        from_address=transfer_in.from_address,
-        to_address=transfer_out.to_address,
-        token_in_address=transfer_in.token_address,
-        token_in_amount=transfer_in.amount,
-        token_out_address=transfer_out.token_address,
-        token_out_amount=transfer_out.amount,
-        error=trace.error,
-    )
-
-
-def _get_recipient_address(trace: DecodedCallTrace) -> Optional[str]:
     classifier = get_classifier(trace)
     if classifier is not None and issubclass(classifier, SwapClassifier):
-        return classifier.get_swap_recipient(trace)
+        return classifier.parse_swap(trace, prior_transfers, child_transfers)
-
     return None

mev_inspect/tokenflow.py

@@ -1,6 +1,7 @@
 from typing import List, Optional
 
-from mev_inspect.schemas import Block, Trace, TraceType
+from mev_inspect.schemas.blocks import Block
+from mev_inspect.schemas.traces import Trace, TraceType
 
 weth_address = "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2"
 

mev_inspect/traces.py

@@ -1,7 +1,7 @@
 from itertools import groupby
 from typing import Dict, List
 
-from mev_inspect.schemas.classified_traces import ClassifiedTrace
+from mev_inspect.schemas.traces import ClassifiedTrace
 
 
 def is_child_trace_address(

mev_inspect/transfers.py

@@ -2,7 +2,7 @@ from typing import Dict, List, Optional, Sequence
 
 from mev_inspect.classifiers.specs import get_classifier
 from mev_inspect.schemas.classifiers import TransferClassifier
-from mev_inspect.schemas.classified_traces import (
+from mev_inspect.schemas.traces import (
     ClassifiedTrace,
     DecodedCallTrace,
 )

poetry.lock

@@ -1,21 +1,33 @@
 [[package]]
 name = "aiohttp"
-version = "3.7.4.post0"
+version = "3.8.0"
 description = "Async http client/server framework (asyncio)"
 category = "main"
 optional = false
 python-versions = ">=3.6"
 
 [package.dependencies]
-async-timeout = ">=3.0,<4.0"
+aiosignal = ">=1.1.2"
+async-timeout = ">=4.0.0a3,<5.0"
 attrs = ">=17.3.0"
-chardet = ">=2.0,<5.0"
+charset-normalizer = ">=2.0,<3.0"
+frozenlist = ">=1.1.1"
 multidict = ">=4.5,<7.0"
-typing-extensions = ">=3.6.5"
 yarl = ">=1.0,<2.0"
 
 [package.extras]
-speedups = ["aiodns", "brotlipy", "cchardet"]
+speedups = ["aiodns", "brotli", "cchardet"]
+
+[[package]]
+name = "aiosignal"
+version = "1.2.0"
+description = "aiosignal: a list of registered asynchronous callbacks"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+frozenlist = ">=1.1.0"
 
 [[package]]
 name = "alembic"
@@ -45,11 +57,14 @@ wrapt = ">=1.11,<1.13"
 
 [[package]]
 name = "async-timeout"
-version = "3.0.1"
+version = "4.0.0"
 description = "Timeout context manager for asyncio programs"
 category = "main"
 optional = false
-python-versions = ">=3.5.3"
+python-versions = ">=3.6"
+
+[package.dependencies]
+typing-extensions = ">=3.6.5"
 
 [[package]]
 name = "asyncio"
@@ -136,14 +151,6 @@ category = "dev"
 optional = false
 python-versions = ">=3.6.1"
 
-[[package]]
-name = "chardet"
-version = "4.0.0"
-description = "Universal encoding detector for Python 2 and 3"
-category = "main"
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-
 [[package]]
 name = "charset-normalizer"
 version = "2.0.4"
@@ -376,6 +383,14 @@ category = "dev"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "frozenlist"
+version = "1.2.0"
+description = "A list-like structure which implements collections.abc.MutableSequence"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
 [[package]]
 name = "greenlet"
 version = "1.1.1"
@@ -1025,47 +1040,86 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.9"
-content-hash = "c9435e8660dcaddeb63b19f26dddb70287b0f3b4e43ca4ad6168d5f919f0089d"
+content-hash = "03aa2d5981665ade1b81682c1e797a06b56c5fb68d61ae69fd2f1e95bd32cfb6"
 
 [metadata.files]
 aiohttp = [
-    {file = "aiohttp-3.7.4.post0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:3cf75f7cdc2397ed4442594b935a11ed5569961333d49b7539ea741be2cc79d5"},
+    {file = "aiohttp-3.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:48f218a5257b6bc16bcf26a91d97ecea0c7d29c811a90d965f3dd97c20f016d6"},
-    {file = "aiohttp-3.7.4.post0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:4b302b45040890cea949ad092479e01ba25911a15e648429c7c5aae9650c67a8"},
+    {file = "aiohttp-3.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a2fee4d656a7cc9ab47771b2a9e8fad8a9a33331c1b59c3057ecf0ac858f5bfe"},
-    {file = "aiohttp-3.7.4.post0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:fe60131d21b31fd1a14bd43e6bb88256f69dfc3188b3a89d736d6c71ed43ec95"},
+    {file = "aiohttp-3.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:688a1eb8c1a5f7e795c7cb67e0fe600194e6723ba35f138dfae0db20c0cb8f94"},
-    {file = "aiohttp-3.7.4.post0-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:393f389841e8f2dfc86f774ad22f00923fdee66d238af89b70ea314c4aefd290"},
+    {file = "aiohttp-3.8.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ba09bb3dcb0b7ec936a485db2b64be44fe14cdce0a5eac56f50e55da3627385"},
-    {file = "aiohttp-3.7.4.post0-cp36-cp36m-manylinux2014_ppc64le.whl", hash = "sha256:c6e9dcb4cb338d91a73f178d866d051efe7c62a7166653a91e7d9fb18274058f"},
+    {file = "aiohttp-3.8.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d7715daf84f10bcebc083ad137e3eced3e1c8e7fa1f096ade9a8d02b08f0d91c"},
-    {file = "aiohttp-3.7.4.post0-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:5df68496d19f849921f05f14f31bd6ef53ad4b00245da3195048c69934521809"},
+    {file = "aiohttp-3.8.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e3f81fbbc170418e22918a9585fd7281bbc11d027064d62aa4b507552c92671"},
-    {file = "aiohttp-3.7.4.post0-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:0563c1b3826945eecd62186f3f5c7d31abb7391fedc893b7e2b26303b5a9f3fe"},
+    {file = "aiohttp-3.8.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1fa9f50aa1f114249b7963c98e20dc35c51be64096a85bc92433185f331de9cc"},
-    {file = "aiohttp-3.7.4.post0-cp36-cp36m-win32.whl", hash = "sha256:3d78619672183be860b96ed96f533046ec97ca067fd46ac1f6a09cd9b7484287"},
+    {file = "aiohttp-3.8.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8a50150419b741ee048b53146c39c47053f060cb9d98e78be08fdbe942eaa3c4"},
-    {file = "aiohttp-3.7.4.post0-cp36-cp36m-win_amd64.whl", hash = "sha256:f705e12750171c0ab4ef2a3c76b9a4024a62c4103e3a55dd6f99265b9bc6fcfc"},
+    {file = "aiohttp-3.8.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a84c335337b676d832c1e2bc47c3a97531b46b82de9f959dafb315cbcbe0dfcd"},
-    {file = "aiohttp-3.7.4.post0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:230a8f7e24298dea47659251abc0fd8b3c4e38a664c59d4b89cca7f6c09c9e87"},
+    {file = "aiohttp-3.8.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:88d4917c30fcd7f6404fb1dc713fa21de59d3063dcc048f4a8a1a90e6bbbd739"},
-    {file = "aiohttp-3.7.4.post0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:2e19413bf84934d651344783c9f5e22dee452e251cfd220ebadbed2d9931dbf0"},
+    {file = "aiohttp-3.8.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b76669b7c058b8020b11008283c3b8e9c61bfd978807c45862956119b77ece45"},
-    {file = "aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:e4b2b334e68b18ac9817d828ba44d8fcb391f6acb398bcc5062b14b2cbeac970"},
+    {file = "aiohttp-3.8.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:84fe1732648c1bc303a70faa67cbc2f7f2e810c8a5bca94f6db7818e722e4c0a"},
-    {file = "aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:d012ad7911653a906425d8473a1465caa9f8dea7fcf07b6d870397b774ea7c0f"},
+    {file = "aiohttp-3.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:730b7c2b7382194d9985ffdc32ab317e893bca21e0665cb1186bdfbb4089d990"},
-    {file = "aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:40eced07f07a9e60e825554a31f923e8d3997cfc7fb31dbc1328c70826e04cde"},
+    {file = "aiohttp-3.8.0-cp310-cp310-win32.whl", hash = "sha256:0a96473a1f61d7920a9099bc8e729dc8282539d25f79c12573ee0fdb9c8b66a8"},
-    {file = "aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:209b4a8ee987eccc91e2bd3ac36adee0e53a5970b8ac52c273f7f8fd4872c94c"},
+    {file = "aiohttp-3.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:764c7c6aa1f78bd77bd9674fc07d1ec44654da1818d0eef9fb48aa8371a3c847"},
-    {file = "aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:14762875b22d0055f05d12abc7f7d61d5fd4fe4642ce1a249abdf8c700bf1fd8"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9951c2696c4357703001e1fe6edc6ae8e97553ac630492ea1bf64b429cb712a3"},
-    {file = "aiohttp-3.7.4.post0-cp37-cp37m-win32.whl", hash = "sha256:7615dab56bb07bff74bc865307aeb89a8bfd9941d2ef9d817b9436da3a0ea54f"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0af379221975054162959e00daf21159ff69a712fc42ed0052caddbd70d52ff4"},
-    {file = "aiohttp-3.7.4.post0-cp37-cp37m-win_amd64.whl", hash = "sha256:d9e13b33afd39ddeb377eff2c1c4f00544e191e1d1dee5b6c51ddee8ea6f0cf5"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9689af0f0a89e5032426c143fa3683b0451f06c83bf3b1e27902bd33acfae769"},
-    {file = "aiohttp-3.7.4.post0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:547da6cacac20666422d4882cfcd51298d45f7ccb60a04ec27424d2f36ba3eaf"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe4a327da0c6b6e59f2e474ae79d6ee7745ac3279fd15f200044602fa31e3d79"},
-    {file = "aiohttp-3.7.4.post0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:af9aa9ef5ba1fd5b8c948bb11f44891968ab30356d65fd0cc6707d989cd521df"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ecb314e59bedb77188017f26e6b684b1f6d0465e724c3122a726359fa62ca1ba"},
-    {file = "aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:64322071e046020e8797117b3658b9c2f80e3267daec409b350b6a7a05041213"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a5399a44a529083951b55521cf4ecbf6ad79fd54b9df57dbf01699ffa0549fc9"},
-    {file = "aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:bb437315738aa441251214dad17428cafda9cdc9729499f1d6001748e1d432f4"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:09754a0d5eaab66c37591f2f8fac8f9781a5f61d51aa852a3261c4805ca6b984"},
-    {file = "aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:e54962802d4b8b18b6207d4a927032826af39395a3bd9196a5af43fc4e60b009"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:adf0cb251b1b842c9dee5cfcdf880ba0aae32e841b8d0e6b6feeaef002a267c5"},
-    {file = "aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:a00bb73540af068ca7390e636c01cbc4f644961896fa9363154ff43fd37af2f5"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:a4759e85a191de58e0ea468ab6fd9c03941986eee436e0518d7a9291fab122c8"},
-    {file = "aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:79ebfc238612123a713a457d92afb4096e2148be17df6c50fb9bf7a81c2f8013"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:28369fe331a59d80393ec82df3d43307c7461bfaf9217999e33e2acc7984ff7c"},
-    {file = "aiohttp-3.7.4.post0-cp38-cp38-win32.whl", hash = "sha256:515dfef7f869a0feb2afee66b957cc7bbe9ad0cdee45aec7fdc623f4ecd4fb16"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2f44d1b1c740a9e2275160d77c73a11f61e8a916191c572876baa7b282bcc934"},
-    {file = "aiohttp-3.7.4.post0-cp38-cp38-win_amd64.whl", hash = "sha256:114b281e4d68302a324dd33abb04778e8557d88947875cbf4e842c2c01a030c5"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-win32.whl", hash = "sha256:e27cde1e8d17b09730801ce97b6e0c444ba2a1f06348b169fd931b51d3402f0d"},
-    {file = "aiohttp-3.7.4.post0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:7b18b97cf8ee5452fa5f4e3af95d01d84d86d32c5e2bfa260cf041749d66360b"},
+    {file = "aiohttp-3.8.0-cp36-cp36m-win_amd64.whl", hash = "sha256:15a660d06092b7c92ed17c1dbe6c1eab0a02963992d60e3e8b9d5fa7fa81f01e"},
-    {file = "aiohttp-3.7.4.post0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:15492a6368d985b76a2a5fdd2166cddfea5d24e69eefed4630cbaae5c81d89bd"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:257f4fad1714d26d562572095c8c5cd271d5a333252795cb7a002dca41fdbad7"},
-    {file = "aiohttp-3.7.4.post0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bdb230b4943891321e06fc7def63c7aace16095be7d9cf3b1e01be2f10fba439"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6074a3b2fa2d0c9bf0963f8dfc85e1e54a26114cc8594126bc52d3fa061c40e"},
-    {file = "aiohttp-3.7.4.post0-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:cffe3ab27871bc3ea47df5d8f7013945712c46a3cc5a95b6bee15887f1675c22"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7a315ceb813208ef32bdd6ec3a85cbe3cb3be9bbda5fd030c234592fa9116993"},
-    {file = "aiohttp-3.7.4.post0-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:f881853d2643a29e643609da57b96d5f9c9b93f62429dcc1cbb413c7d07f0e1a"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9a52b141ff3b923a9166595de6e3768a027546e75052ffba267d95b54267f4ab"},
-    {file = "aiohttp-3.7.4.post0-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:a5ca29ee66f8343ed336816c553e82d6cade48a3ad702b9ffa6125d187e2dedb"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6a038cb1e6e55b26bb5520ccffab7f539b3786f5553af2ee47eb2ec5cbd7084e"},
-    {file = "aiohttp-3.7.4.post0-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:17c073de315745a1510393a96e680d20af8e67e324f70b42accbd4cb3315c9fb"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98b1ea2763b33559dd9ec621d67fc17b583484cb90735bfb0ec3614c17b210e4"},
-    {file = "aiohttp-3.7.4.post0-cp39-cp39-win32.whl", hash = "sha256:932bb1ea39a54e9ea27fc9232163059a0b8855256f4052e776357ad9add6f1c9"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9e8723c3256641e141cd18f6ce478d54a004138b9f1a36e41083b36d9ecc5fc5"},
-    {file = "aiohttp-3.7.4.post0-cp39-cp39-win_amd64.whl", hash = "sha256:02f46fc0e3c5ac58b80d4d56eb0a7c7d97fcef69ace9326289fb9f1955e65cfe"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:14a6f026eca80dfa3d52e86be89feb5cd878f6f4a6adb34457e2c689fd85229b"},
-    {file = "aiohttp-3.7.4.post0.tar.gz", hash = "sha256:493d3299ebe5f5a7c66b9819eacdcfbbaaf1a8e84911ddffcdc48888497afecf"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c62d4791a8212c885b97a63ef5f3974b2cd41930f0cd224ada9c6ee6654f8150"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:90a97c2ed2830e7974cbe45f0838de0aefc1c123313f7c402e21c29ec063fbb4"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:dcc4d5dd5fba3affaf4fd08f00ef156407573de8c63338787614ccc64f96b321"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-win32.whl", hash = "sha256:de42f513ed7a997bc821bddab356b72e55e8396b1b7ba1bf39926d538a76a90f"},
+    {file = "aiohttp-3.8.0-cp37-cp37m-win_amd64.whl", hash = "sha256:7d76e8a83396e06abe3df569b25bd3fc88bf78b7baa2c8e4cf4aaf5983af66a3"},
+    {file = "aiohttp-3.8.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5d79174d96446a02664e2bffc95e7b6fa93b9e6d8314536c5840dff130d0878b"},
+    {file = "aiohttp-3.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4a6551057a846bf72c7a04f73de3fcaca269c0bd85afe475ceb59d261c6a938c"},
+    {file = "aiohttp-3.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:871d4fdc56288caa58b1094c20f2364215f7400411f76783ea19ad13be7c8e19"},
+    {file = "aiohttp-3.8.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ba08a71caa42eef64357257878fb17f3fba3fba6e81a51d170e32321569e079"},
+    {file = "aiohttp-3.8.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:51f90dabd9933b1621260b32c2f0d05d36923c7a5a909eb823e429dba0fd2f3e"},
+    {file = "aiohttp-3.8.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f348ebd20554e8bc26e8ef3ed8a134110c0f4bf015b3b4da6a4ddf34e0515b19"},
+    {file = "aiohttp-3.8.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d5f8c04574efa814a24510122810e3a3c77c0552f9f6ff65c9862f1f046be2c3"},
+    {file = "aiohttp-3.8.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5ecffdc748d3b40dd3618ede0170e4f5e1d3c9647cfb410d235d19e62cb54ee0"},
+    {file = "aiohttp-3.8.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:577cc2c7b807b174814dac2d02e673728f2e46c7f90ceda3a70ea4bb6d90b769"},
+    {file = "aiohttp-3.8.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6b79f6c31e68b6dafc0317ec453c83c86dd8db1f8f0c6f28e97186563fca87a0"},
+    {file = "aiohttp-3.8.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2bdd655732e38b40f8a8344d330cfae3c727fb257585df923316aabbd489ccb8"},
+    {file = "aiohttp-3.8.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:63fa57a0708573d3c059f7b5527617bd0c291e4559298473df238d502e4ab98c"},
+    {file = "aiohttp-3.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d3f90ee275b1d7c942e65b5c44c8fb52d55502a0b9a679837d71be2bd8927661"},
+    {file = "aiohttp-3.8.0-cp38-cp38-win32.whl", hash = "sha256:fa818609357dde5c4a94a64c097c6404ad996b1d38ca977a72834b682830a722"},
+    {file = "aiohttp-3.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:097ecf52f6b9859b025c1e36401f8aa4573552e887d1b91b4b999d68d0b5a3b3"},
+    {file = "aiohttp-3.8.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:be03a7483ad9ea60388f930160bb3728467dd0af538aa5edc60962ee700a0bdc"},
+    {file = "aiohttp-3.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:78d51e35ed163783d721b6f2ce8ce3f82fccfe471e8e50a10fba13a766d31f5a"},
+    {file = "aiohttp-3.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bda75d73e7400e81077b0910c9a60bf9771f715420d7e35fa7739ae95555f195"},
+    {file = "aiohttp-3.8.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:707adc30ea6918fba725c3cb3fe782d271ba352b22d7ae54a7f9f2e8a8488c41"},
+    {file = "aiohttp-3.8.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f58aa995b905ab82fe228acd38538e7dc1509e01508dcf307dad5046399130f"},
+    {file = "aiohttp-3.8.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c996eb91bfbdab1e01e2c02e7ff678c51e2b28e3a04e26e41691991cc55795"},
+    {file = "aiohttp-3.8.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d6a1a66bb8bac9bc2892c2674ea363486bfb748b86504966a390345a11b1680e"},
+    {file = "aiohttp-3.8.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:dafc01a32b4a1d7d3ef8bfd3699406bb44f7b2e0d3eb8906d574846e1019b12f"},
+    {file = "aiohttp-3.8.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:949a605ef3907254b122f845baa0920407080cdb1f73aa64f8d47df4a7f4c4f9"},
+    {file = "aiohttp-3.8.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0d7b056fd3972d353cb4bc305c03f9381583766b7f8c7f1c44478dba69099e33"},
+    {file = "aiohttp-3.8.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f1d39a744101bf4043fa0926b3ead616607578192d0a169974fb5265ab1e9d2"},
+    {file = "aiohttp-3.8.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:67ca7032dfac8d001023fadafc812d9f48bf8a8c3bb15412d9cdcf92267593f4"},
+    {file = "aiohttp-3.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cb751ef712570d3bda9a73fd765ff3e1aba943ec5d52a54a0c2e89c7eef9da1e"},
+    {file = "aiohttp-3.8.0-cp39-cp39-win32.whl", hash = "sha256:6d3e027fe291b77f6be9630114a0200b2c52004ef20b94dc50ca59849cd623b3"},
+    {file = "aiohttp-3.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:3c5e9981e449d54308c6824f172ec8ab63eb9c5f922920970249efee83f7e919"},
+    {file = "aiohttp-3.8.0.tar.gz", hash = "sha256:d3b19d8d183bcfd68b25beebab8dc3308282fe2ca3d6ea3cb4cd101b3c279f8d"},
+]
+aiosignal = [
+    {file = "aiosignal-1.2.0-py3-none-any.whl", hash = "sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a"},
+    {file = "aiosignal-1.2.0.tar.gz", hash = "sha256:78ed67db6c7b7ced4f98e495e572106d5c432a93e1ddd1bf475e1dc05f5b7df2"},
 ]
 alembic = [
     {file = "alembic-1.6.5-py2.py3-none-any.whl", hash = "sha256:e78be5b919f5bb184e3e0e2dd1ca986f2362e29a2bc933c446fe89f39dbe4e9c"},
@@ -1076,8 +1130,8 @@ astroid = [
     {file = "astroid-2.7.2.tar.gz", hash = "sha256:b6c2d75cd7c2982d09e7d41d70213e863b3ba34d3bd4014e08f167cee966e99e"},
 ]
 async-timeout = [
-    {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"},
+    {file = "async-timeout-4.0.0.tar.gz", hash = "sha256:7d87a4e8adba8ededb52e579ce6bc8276985888913620c935094c2276fd83382"},
-    {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"},
+    {file = "async_timeout-4.0.0-py3-none-any.whl", hash = "sha256:f3303dddf6cafa748a92747ab6c2ecf60e0aeca769aee4c151adfce243a05d9b"},
 ]
 asyncio = [
     {file = "asyncio-3.4.3-cp33-none-win32.whl", hash = "sha256:b62c9157d36187eca799c378e572c969f0da87cd5fc42ca372d92cdb06e7e1de"},
@@ -1116,10 +1170,6 @@ cfgv = [
     {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"},
     {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"},
 ]
-chardet = [
-    {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"},
-    {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"},
-]
 charset-normalizer = [
     {file = "charset-normalizer-2.0.4.tar.gz", hash = "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"},
     {file = "charset_normalizer-2.0.4-py3-none-any.whl", hash = "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b"},
@@ -1252,6 +1302,80 @@ filelock = [
     {file = "filelock-3.0.12-py3-none-any.whl", hash = "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"},
     {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"},
 ]
+frozenlist = [
+    {file = "frozenlist-1.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:977a1438d0e0d96573fd679d291a1542097ea9f4918a8b6494b06610dfeefbf9"},
+    {file = "frozenlist-1.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a8d86547a5e98d9edd47c432f7a14b0c5592624b496ae9880fb6332f34af1edc"},
+    {file = "frozenlist-1.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:181754275d5d32487431a0a29add4f897968b7157204bc1eaaf0a0ce80c5ba7d"},
+    {file = "frozenlist-1.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5df31bb2b974f379d230a25943d9bf0d3bc666b4b0807394b131a28fca2b0e5f"},
+    {file = "frozenlist-1.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4766632cd8a68e4f10f156a12c9acd7b1609941525569dd3636d859d79279ed3"},
+    {file = "frozenlist-1.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16eef427c51cb1203a7c0ab59d1b8abccaba9a4f58c4bfca6ed278fc896dc193"},
+    {file = "frozenlist-1.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:01d79515ed5aa3d699b05f6bdcf1fe9087d61d6b53882aa599a10853f0479c6c"},
+    {file = "frozenlist-1.2.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:28e164722ea0df0cf6d48c4d5bdf3d19e87aaa6dfb39b0ba91153f224b912020"},
+    {file = "frozenlist-1.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e63ad0beef6ece06475d29f47d1f2f29727805376e09850ebf64f90777962792"},
+    {file = "frozenlist-1.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:41de4db9b9501679cf7cddc16d07ac0f10ef7eb58c525a1c8cbff43022bddca4"},
+    {file = "frozenlist-1.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c6a9d84ee6427b65a81fc24e6ef589cb794009f5ca4150151251c062773e7ed2"},
+    {file = "frozenlist-1.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:f5f3b2942c3b8b9bfe76b408bbaba3d3bb305ee3693e8b1d631fe0a0d4f93673"},
+    {file = "frozenlist-1.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c98d3c04701773ad60d9545cd96df94d955329efc7743fdb96422c4b669c633b"},
+    {file = "frozenlist-1.2.0-cp310-cp310-win32.whl", hash = "sha256:72cfbeab7a920ea9e74b19aa0afe3b4ad9c89471e3badc985d08756efa9b813b"},
+    {file = "frozenlist-1.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:11ff401951b5ac8c0701a804f503d72c048173208490c54ebb8d7bb7c07a6d00"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b46f997d5ed6d222a863b02cdc9c299101ee27974d9bbb2fd1b3c8441311c408"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:351686ca020d1bcd238596b1fa5c8efcbc21bffda9d0efe237aaa60348421e2a"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfbaa08cf1452acad9cb1c1d7b89394a41e712f88df522cea1a0f296b57782a0"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2ae2f5e9fa10805fb1c9adbfefaaecedd9e31849434be462c3960a0139ed729"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6790b8d96bbb74b7a6f4594b6f131bd23056c25f2aa5d816bd177d95245a30e3"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:41f62468af1bd4e4b42b5508a3fe8cc46a693f0cdd0ca2f443f51f207893d837"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:ec6cf345771cdb00791d271af9a0a6fbfc2b6dd44cb753f1eeaa256e21622adb"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:14a5cef795ae3e28fb504b73e797c1800e9249f950e1c964bb6bdc8d77871161"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:8b54cdd2fda15467b9b0bfa78cee2ddf6dbb4585ef23a16e14926f4b076dfae4"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:f025f1d6825725b09c0038775acab9ae94264453a696cc797ce20c0769a7b367"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:84e97f59211b5b9083a2e7a45abf91cfb441369e8bb6d1f5287382c1c526def3"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-win32.whl", hash = "sha256:c5328ed53fdb0a73c8a50105306a3bc013e5ca36cca714ec4f7bd31d38d8a97f"},
+    {file = "frozenlist-1.2.0-cp36-cp36m-win_amd64.whl", hash = "sha256:9ade70aea559ca98f4b1b1e5650c45678052e76a8ab2f76d90f2ac64180215a2"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0d3ffa8772464441b52489b985d46001e2853a3b082c655ec5fad9fb6a3d618"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3457f8cf86deb6ce1ba67e120f1b0128fcba1332a180722756597253c465fc1d"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a72eecf37eface331636951249d878750db84034927c997d47f7f78a573b72b"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:acc4614e8d1feb9f46dd829a8e771b8f5c4b1051365d02efb27a3229048ade8a"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:87521e32e18a2223311afc2492ef2d99946337da0779ddcda77b82ee7319df59"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8b4c7665a17c3a5430edb663e4ad4e1ad457614d1b2f2b7f87052e2ef4fa45ca"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ed58803563a8c87cf4c0771366cf0ad1aa265b6b0ae54cbbb53013480c7ad74d"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:aa44c4740b4e23fcfa259e9dd52315d2b1770064cde9507457e4c4a65a04c397"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:2de5b931701257d50771a032bba4e448ff958076380b049fd36ed8738fdb375b"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:6e105013fa84623c057a4381dc8ea0361f4d682c11f3816cc80f49a1f3bc17c6"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:705c184b77565955a99dc360f359e8249580c6b7eaa4dc0227caa861ef46b27a"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-win32.whl", hash = "sha256:a37594ad6356e50073fe4f60aa4187b97d15329f2138124d252a5a19c8553ea4"},
+    {file = "frozenlist-1.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:25b358aaa7dba5891b05968dd539f5856d69f522b6de0bf34e61f133e077c1a4"},
+    {file = "frozenlist-1.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af2a51c8a381d76eabb76f228f565ed4c3701441ecec101dd18be70ebd483cfd"},
+    {file = "frozenlist-1.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:82d22f6e6f2916e837c91c860140ef9947e31194c82aaeda843d6551cec92f19"},
+    {file = "frozenlist-1.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1cfe6fef507f8bac40f009c85c7eddfed88c1c0d38c75e72fe10476cef94e10f"},
+    {file = "frozenlist-1.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26f602e380a5132880fa245c92030abb0fc6ff34e0c5500600366cedc6adb06a"},
+    {file = "frozenlist-1.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ad065b2ebd09f32511ff2be35c5dfafee6192978b5a1e9d279a5c6e121e3b03"},
+    {file = "frozenlist-1.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc93f5f62df3bdc1f677066327fc81f92b83644852a31c6aa9b32c2dde86ea7d"},
+    {file = "frozenlist-1.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:89fdfc84c6bf0bff2ff3170bb34ecba8a6911b260d318d377171429c4be18c73"},
+    {file = "frozenlist-1.2.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:47b2848e464883d0bbdcd9493c67443e5e695a84694efff0476f9059b4cb6257"},
+    {file = "frozenlist-1.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4f52d0732e56906f8ddea4bd856192984650282424049c956857fed43697ea43"},
+    {file = "frozenlist-1.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:16ef7dd5b7d17495404a2e7a49bac1bc13d6d20c16d11f4133c757dd94c4144c"},
+    {file = "frozenlist-1.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1cf63243bc5f5c19762943b0aa9e0d3fb3723d0c514d820a18a9b9a5ef864315"},
+    {file = "frozenlist-1.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:54a1e09ab7a69f843cd28fefd2bcaf23edb9e3a8d7680032c8968b8ac934587d"},
+    {file = "frozenlist-1.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:954b154a4533ef28bd3e83ffdf4eadf39deeda9e38fb8feaf066d6069885e034"},
+    {file = "frozenlist-1.2.0-cp38-cp38-win32.whl", hash = "sha256:cb3957c39668d10e2b486acc85f94153520a23263b6401e8f59422ef65b9520d"},
+    {file = "frozenlist-1.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:0a7c7cce70e41bc13d7d50f0e5dd175f14a4f1837a8549b0936ed0cbe6170bf9"},
+    {file = "frozenlist-1.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4c457220468d734e3077580a3642b7f682f5fd9507f17ddf1029452450912cdc"},
+    {file = "frozenlist-1.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e74f8b4d8677ebb4015ac01fcaf05f34e8a1f22775db1f304f497f2f88fdc697"},
+    {file = "frozenlist-1.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fbd4844ff111449f3bbe20ba24fbb906b5b1c2384d0f3287c9f7da2354ce6d23"},
+    {file = "frozenlist-1.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0081a623c886197ff8de9e635528fd7e6a387dccef432149e25c13946cb0cd0"},
+    {file = "frozenlist-1.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9b6e21e5770df2dea06cb7b6323fbc008b13c4a4e3b52cb54685276479ee7676"},
+    {file = "frozenlist-1.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:406aeb340613b4b559db78d86864485f68919b7141dec82aba24d1477fd2976f"},
+    {file = "frozenlist-1.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:878ebe074839d649a1cdb03a61077d05760624f36d196884a5cafb12290e187b"},
+    {file = "frozenlist-1.2.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1fef737fd1388f9b93bba8808c5f63058113c10f4e3c0763ced68431773f72f9"},
+    {file = "frozenlist-1.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4a495c3d513573b0b3f935bfa887a85d9ae09f0627cf47cad17d0cc9b9ba5c38"},
+    {file = "frozenlist-1.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e7d0dd3e727c70c2680f5f09a0775525229809f1a35d8552b92ff10b2b14f2c2"},
+    {file = "frozenlist-1.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:66a518731a21a55b7d3e087b430f1956a36793acc15912e2878431c7aec54210"},
+    {file = "frozenlist-1.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:94728f97ddf603d23c8c3dd5cae2644fa12d33116e69f49b1644a71bb77b89ae"},
+    {file = "frozenlist-1.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c1e8e9033d34c2c9e186e58279879d78c94dd365068a3607af33f2bc99357a53"},
+    {file = "frozenlist-1.2.0-cp39-cp39-win32.whl", hash = "sha256:83334e84a290a158c0c4cc4d22e8c7cfe0bba5b76d37f1c2509dabd22acafe15"},
+    {file = "frozenlist-1.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:735f386ec522e384f511614c01d2ef9cf799f051353876b4c6fb93ef67a6d1ee"},
+    {file = "frozenlist-1.2.0.tar.gz", hash = "sha256:68201be60ac56aff972dc18085800b6ee07973c49103a8aba669dee3d71079de"},
+]
 greenlet = [
     {file = "greenlet-1.1.1-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:476ba9435afaead4382fbab8f1882f75e3fb2285c35c9285abb3dd30237f9142"},
     {file = "greenlet-1.1.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:44556302c0ab376e37939fd0058e1f0db2e769580d340fb03b01678d1ff25f68"},

pyproject.toml

@@ -11,7 +11,7 @@ pydantic = "^1.8.2"
 hexbytes = "^0.2.1"
 click = "^8.0.1"
 psycopg2 = "^2.9.1"
-aiohttp = "^3.7.4"
+aiohttp = "^3.8.0"
 asyncio = "^3.4.3"
 
 [tool.poetry.dev-dependencies]
@@ -34,6 +34,7 @@ build-backend = "poetry.core.masonry.api"
 [tool.poetry.scripts]
 inspect-block = 'cli:inspect_block_command'
 inspect-many-blocks = 'cli:inspect_many_blocks_command'
+fetch-block = 'cli:fetch_block_command'
 
 [tool.black]
 exclude = '''

tests/helpers.py

@@ -1,11 +1,11 @@
 from typing import List, Optional
 
-from mev_inspect.schemas.blocks import TraceType
+from mev_inspect.schemas.traces import (
-from mev_inspect.schemas.classified_traces import (
     Classification,
     ClassifiedTrace,
     DecodedCallTrace,
     Protocol,
+    TraceType,
 )
 
 
@@ -44,7 +44,7 @@ def make_swap_trace(
     transaction_hash: str,
     trace_address: List[int],
     from_address: str,
-    pool_address: str,
+    contract_address: str,
     abi_name: str,
     function_signature: str,
     protocol: Optional[Protocol],
@@ -60,7 +60,7 @@ def make_swap_trace(
         subtraces=0,
         classification=Classification.swap,
         from_address=from_address,
-        to_address=pool_address,
+        to_address=contract_address,
         function_name="swap",
         function_signature=function_signature,
         inputs={recipient_input_key: recipient_address},

tests/liquidation_test.py

@@ -2,8 +2,9 @@ from typing import List
 
 from mev_inspect.aave_liquidations import get_aave_liquidations
 from mev_inspect.schemas.liquidations import Liquidation
-from mev_inspect.schemas.classified_traces import Protocol
+from mev_inspect.schemas.traces import Protocol
 from mev_inspect.classifiers.trace import TraceClassifier
+from mev_inspect.transfers import ETH_TOKEN_ADDRESS
 from tests.utils import load_test_block
 
 
@@ -158,6 +159,50 @@ def test_multiple_liquidations_in_block():
     _assert_equal_list_of_liquidations(result, liquidations)
 
 
+def test_liquidations_with_eth_transfer():
+
+    transaction_hash = (
+        "0xf687fedbc4bbc25adb3ef3a35c20c38fb7d35d86d7633d5061d2e3c4f86311b7"
+    )
+    block_number = 13302365
+
+    liquidation1 = Liquidation(
+        liquidated_user="0xad346c7762f74c78da86d2941c6eb546e316fbd0",
+        liquidator_user="0x27239549dd40e1d60f5b80b0c4196923745b1fd2",
+        collateral_token_address=ETH_TOKEN_ADDRESS,
+        debt_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
+        debt_purchase_amount=1809152000000000000,
+        received_amount=15636807387264000,
+        received_token_address=ETH_TOKEN_ADDRESS,
+        protocol=Protocol.aave,
+        transaction_hash=transaction_hash,
+        trace_address=[2, 3, 2],
+        block_number=block_number,
+    )
+
+    liquidation2 = Liquidation(
+        liquidated_user="0xad346c7762f74c78da86d2941c6eb546e316fbd0",
+        liquidator_user="0x27239549dd40e1d60f5b80b0c4196923745b1fd2",
+        collateral_token_address=ETH_TOKEN_ADDRESS,
+        debt_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
+        debt_purchase_amount=1809152000000000000,
+        received_amount=8995273139160873,
+        received_token_address=ETH_TOKEN_ADDRESS,
+        protocol=Protocol.aave,
+        transaction_hash=transaction_hash,
+        trace_address=[2, 4, 2],
+        block_number=block_number,
+    )
+
+    block = load_test_block(block_number)
+    trace_classifier = TraceClassifier()
+    classified_traces = trace_classifier.classify(block.traces)
+    result = get_aave_liquidations(classified_traces)
+    liquidations = [liquidation1, liquidation2]
+
+    _assert_equal_list_of_liquidations(result, liquidations)
+
+
 def _assert_equal_list_of_liquidations(
     actual_liquidations: List[Liquidation], expected_liquidations: List[Liquidation]
 ):

tests/test_arbitrage_integration.py

@@ -8,19 +8,54 @@ from .utils import load_test_block
 def test_arbitrage_real_block():
     block = load_test_block(12914944)
 
-    trace_clasifier = TraceClassifier()
+    trace_classifier = TraceClassifier()
-    classified_traces = trace_clasifier.classify(block.traces)
+    classified_traces = trace_classifier.classify(block.traces)
 
     swaps = get_swaps(classified_traces)
     assert len(swaps) == 51
 
     arbitrages = get_arbitrages(list(swaps))
-    assert len(arbitrages) == 1
+    assert len(arbitrages) == 2
 
-    arbitrage = arbitrages[0]
+    arbitrage_1 = [
+        arb
+        for arb in arbitrages
+        if arb.transaction_hash
+        == "0x448245bf1a507b73516c4eeee01611927dada6610bf26d403012f2e66800d8f0"
+    ][0]
+    arbitrage_2 = [
+        arb
+        for arb in arbitrages
+        if arb.transaction_hash
+        == "0xfcf4558f6432689ea57737fe63124a5ec39fd6ba6aaf198df13a825dd599bffc"
+    ][0]
 
-    assert len(arbitrage.swaps) == 3
+    assert len(arbitrage_1.swaps) == 3
     assert (
-        arbitrage.profit_token_address == "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2"
+        arbitrage_1.profit_token_address == "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2"
     )
-    assert arbitrage.profit_amount == 53560707941943273628
+    assert len(arbitrage_1.swaps) == 3
+    assert (
+        arbitrage_1.swaps[1].token_in_address
+        == "0x25f8087ead173b73d6e8b84329989a8eea16cf73"
+    )
+    assert (
+        arbitrage_1.swaps[1].token_out_address
+        == "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"
+    )
+    assert arbitrage_1.profit_amount == 750005273675102326
+
+    assert len(arbitrage_2.swaps) == 3
+    assert (
+        arbitrage_2.profit_token_address == "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2"
+    )
+    assert len(arbitrage_2.swaps) == 3
+    assert (
+        arbitrage_2.swaps[1].token_in_address
+        == "0x25f8087ead173b73d6e8b84329989a8eea16cf73"
+    )
+    assert (
+        arbitrage_2.swaps[1].token_out_address
+        == "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"
+    )
+    assert arbitrage_2.profit_amount == 53560707941943273628

tests/test_arbitrages.py

@@ -1,9 +1,11 @@
-from mev_inspect.arbitrages import get_arbitrages
+from typing import List
+
+from mev_inspect.arbitrages import get_arbitrages, _get_all_routes
+from mev_inspect.schemas.swaps import Swap
 from mev_inspect.classifiers.specs.uniswap import (
     UNISWAP_V2_PAIR_ABI_NAME,
     UNISWAP_V3_POOL_ABI_NAME,
 )
-from mev_inspect.schemas.swaps import Swap
 
 
 def test_two_pool_arbitrage(get_transaction_hashes, get_addresses):
@@ -17,10 +19,11 @@ def test_two_pool_arbitrage(get_transaction_hashes, get_addresses):
         unrelated_pool_address,
         first_token_address,
         second_token_address,
-    ] = get_addresses(6)
+        third_token_address,
+    ] = get_addresses(7)
 
     first_token_in_amount = 10
-    first_token_out_amount = 10
+    first_token_out_amount = 11
     second_token_amount = 15
 
     arb_swaps = [
@@ -29,7 +32,7 @@ def test_two_pool_arbitrage(get_transaction_hashes, get_addresses):
             transaction_hash=transaction_hash,
             block_number=block_number,
             trace_address=[0],
-            pool_address=first_pool_address,
+            contract_address=first_pool_address,
             from_address=account_address,
             to_address=second_pool_address,
             token_in_address=first_token_address,
@@ -42,7 +45,7 @@ def test_two_pool_arbitrage(get_transaction_hashes, get_addresses):
             transaction_hash=transaction_hash,
             block_number=block_number,
             trace_address=[1],
-            pool_address=second_pool_address,
+            contract_address=second_pool_address,
             from_address=first_pool_address,
             to_address=account_address,
             token_in_address=second_token_address,
@@ -57,12 +60,12 @@ def test_two_pool_arbitrage(get_transaction_hashes, get_addresses):
         transaction_hash=transaction_hash,
         block_number=block_number,
         trace_address=[2, 0],
-        pool_address=unrelated_pool_address,
+        contract_address=unrelated_pool_address,
         from_address=account_address,
         to_address=account_address,
         token_in_address=second_token_address,
         token_in_amount=first_token_in_amount,
-        token_out_address=first_token_address,
+        token_out_address=third_token_address,
         token_out_amount=first_token_out_amount,
     )
 
@@ -100,7 +103,7 @@ def test_three_pool_arbitrage(get_transaction_hashes, get_addresses):
     ] = get_addresses(7)
 
     first_token_in_amount = 10
-    first_token_out_amount = 10
+    first_token_out_amount = 11
     second_token_amount = 15
     third_token_amount = 40
 
@@ -110,7 +113,7 @@ def test_three_pool_arbitrage(get_transaction_hashes, get_addresses):
             transaction_hash=transaction_hash,
             block_number=block_number,
             trace_address=[0],
-            pool_address=first_pool_address,
+            contract_address=first_pool_address,
             from_address=account_address,
             to_address=second_pool_address,
             token_in_address=first_token_address,
@@ -123,7 +126,7 @@ def test_three_pool_arbitrage(get_transaction_hashes, get_addresses):
             transaction_hash=transaction_hash,
             block_number=block_number,
             trace_address=[1],
-            pool_address=second_pool_address,
+            contract_address=second_pool_address,
             from_address=first_pool_address,
             to_address=third_pool_address,
             token_in_address=second_token_address,
@@ -136,7 +139,7 @@ def test_three_pool_arbitrage(get_transaction_hashes, get_addresses):
             transaction_hash=transaction_hash,
             block_number=block_number,
             trace_address=[2],
-            pool_address=third_pool_address,
+            contract_address=third_pool_address,
             from_address=second_pool_address,
             to_address=account_address,
             token_in_address=third_token_address,
@@ -158,3 +161,70 @@ def test_three_pool_arbitrage(get_transaction_hashes, get_addresses):
     assert arbitrage.start_amount == first_token_in_amount
     assert arbitrage.end_amount == first_token_out_amount
     assert arbitrage.profit_amount == first_token_out_amount - first_token_in_amount
+
+
+def test_get_all_routes():
+    # A -> B, B -> A
+    start_swap = create_generic_swap("0xa", "0xb")
+    end_swap = create_generic_swap("0xb", "0xa")
+    routes = _get_all_routes(start_swap, end_swap, [])
+    assert len(routes) == 1
+
+    # A->B, B->C, C->A
+    start_swap = create_generic_swap("0xa", "0xb")
+    other_swaps = [create_generic_swap("0xb", "0xc")]
+    end_swap = create_generic_swap("0xc", "0xa")
+    routes = _get_all_routes(start_swap, end_swap, other_swaps)
+    assert len(routes) == 1
+
+    # A->B, B->C, C->A + A->D
+    other_swaps.append(create_generic_swap("0xa", "0xd"))
+    routes = _get_all_routes(start_swap, end_swap, other_swaps)
+    assert len(routes) == 1
+
+    # A->B, B->C, C->A + A->D B->E
+    other_swaps.append(create_generic_swap("0xb", "0xe"))
+    routes = _get_all_routes(start_swap, end_swap, other_swaps)
+    assert len(routes) == 1
+
+    # A->B, B->A, B->C, C->A
+    other_swaps = [create_generic_swap("0xb", "0xa"), create_generic_swap("0xb", "0xc")]
+    routes = _get_all_routes(start_swap, end_swap, other_swaps)
+    assert len(routes) == 1
+    expect_simple_route = [["0xa", "0xb"], ["0xb", "0xc"], ["0xc", "0xa"]]
+    assert len(routes[0]) == len(expect_simple_route)
+    for i in range(len(expect_simple_route)):
+        assert expect_simple_route[i][0] == routes[0][i].token_in_address
+        assert expect_simple_route[i][1] == routes[0][i].token_out_address
+
+    # A->B, B->C, C->D, D->A, B->D
+    end_swap = create_generic_swap("0xd", "0xa")
+    other_swaps = [
+        create_generic_swap("0xb", "0xc"),
+        create_generic_swap("0xc", "0xd"),
+        create_generic_swap("0xb", "0xd"),
+    ]
+    routes = _get_all_routes(start_swap, end_swap, other_swaps)
+    assert len(routes) == 2
+
+
+def create_generic_swap(
+    tok_a: str = "0xa",
+    tok_b: str = "0xb",
+    amount_a_in: int = 1,
+    amount_b_out: int = 1,
+    trace_address: List[int] = [],
+):
+    return Swap(
+        abi_name=UNISWAP_V3_POOL_ABI_NAME,
+        transaction_hash="0xfake",
+        block_number=0,
+        trace_address=trace_address,
+        contract_address="0xfake",
+        from_address="0xfake",
+        to_address="0xfake",
+        token_in_address=tok_a,
+        token_in_amount=amount_a_in,
+        token_out_address=tok_b,
+        token_out_amount=amount_b_out,
+    )

tests/test_compound.py

@@ -1,7 +1,8 @@
 from mev_inspect.compound_liquidations import get_compound_liquidations
 from mev_inspect.schemas.liquidations import Liquidation
-from mev_inspect.schemas.classified_traces import Protocol
+from mev_inspect.schemas.traces import Protocol
 from mev_inspect.classifiers.trace import TraceClassifier
+from mev_inspect.transfers import ETH_TOKEN_ADDRESS
 from tests.utils import load_test_block, load_comp_markets, load_cream_markets
 
 comp_markets = load_comp_markets()
@@ -18,7 +19,7 @@ def test_c_ether_liquidations():
         Liquidation(
             liquidated_user="0xb5535a3681cf8d5431b8acfd779e2f79677ecce9",
             liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
-            collateral_token_address="0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE",
+            collateral_token_address=ETH_TOKEN_ADDRESS,
             debt_token_address="0x39aa39c021dfbae8fac545936693ac917d5e7563",
             debt_purchase_amount=268066492249420078,
             received_amount=4747650169097,
@@ -43,7 +44,7 @@ def test_c_ether_liquidations():
         Liquidation(
             liquidated_user="0x45df6f00166c3fb77dc16b9e47ff57bc6694e898",
             liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
-            collateral_token_address="0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE",
+            collateral_token_address=ETH_TOKEN_ADDRESS,
             debt_token_address="0x35a18000230da775cac24873d00ff85bccded550",
             debt_purchase_amount=414547860568297082,
             received_amount=321973320649,
@@ -69,7 +70,7 @@ def test_c_ether_liquidations():
         Liquidation(
             liquidated_user="0xacbcf5d2970eef25f02a27e9d9cd31027b058b9b",
             liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
-            collateral_token_address="0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE",
+            collateral_token_address=ETH_TOKEN_ADDRESS,
             debt_token_address="0x35a18000230da775cac24873d00ff85bccded550",
             debt_purchase_amount=1106497772527562662,
             received_amount=910895850496,

tests/test_swaps.py

@@ -4,7 +4,7 @@ from mev_inspect.classifiers.specs.uniswap import (
     UNISWAP_V2_PAIR_ABI_NAME,
     UNISWAP_V3_POOL_ABI_NAME,
 )
-from mev_inspect.schemas.classified_traces import Protocol
+from mev_inspect.schemas.traces import Protocol
 
 from .helpers import (
     make_unknown_trace,
@@ -63,7 +63,7 @@ def test_swaps(
             first_transaction_hash,
             trace_address=[1],
             from_address=alice_address,
-            pool_address=first_pool_address,
+            contract_address=first_pool_address,
             abi_name=UNISWAP_V2_PAIR_ABI_NAME,
             protocol=None,
             function_signature="swap(uint256,uint256,address,bytes)",
@@ -84,7 +84,7 @@ def test_swaps(
             second_transaction_hash,
             trace_address=[],
             from_address=bob_address,
-            pool_address=second_pool_address,
+            contract_address=second_pool_address,
             abi_name=UNISWAP_V3_POOL_ABI_NAME,
             protocol=None,
             function_signature="swap(address,bool,int256,uint160,bytes)",
@@ -132,7 +132,7 @@ def test_swaps(
             third_transaction_hash,
             trace_address=[6],
             from_address=bob_address,
-            pool_address=third_pool_address,
+            contract_address=third_pool_address,
             abi_name=BALANCER_V1_POOL_ABI_NAME,
             protocol=Protocol.balancer_v1,
             function_signature="swapExactAmountIn(address,uint256,address,uint256,uint256)",
@@ -160,7 +160,7 @@ def test_swaps(
     assert uni_v2_swap.block_number == block_number
     assert uni_v2_swap.trace_address == [1]
     assert uni_v2_swap.protocol is None
-    assert uni_v2_swap.pool_address == first_pool_address
+    assert uni_v2_swap.contract_address == first_pool_address
     assert uni_v2_swap.from_address == alice_address
     assert uni_v2_swap.to_address == bob_address
     assert uni_v2_swap.token_in_address == first_token_in_address
@@ -173,7 +173,7 @@ def test_swaps(
     assert uni_v3_swap.block_number == block_number
     assert uni_v3_swap.trace_address == []
     assert uni_v3_swap.protocol is None
-    assert uni_v3_swap.pool_address == second_pool_address
+    assert uni_v3_swap.contract_address == second_pool_address
     assert uni_v3_swap.from_address == bob_address
     assert uni_v3_swap.to_address == carl_address
     assert uni_v3_swap.token_in_address == second_token_in_address
@@ -186,7 +186,7 @@ def test_swaps(
     assert bal_v1_swap.block_number == block_number
     assert bal_v1_swap.trace_address == [6]
     assert bal_v1_swap.protocol == Protocol.balancer_v1
-    assert bal_v1_swap.pool_address == third_pool_address
+    assert bal_v1_swap.contract_address == third_pool_address
     assert bal_v1_swap.from_address == bob_address
     assert bal_v1_swap.to_address == bob_address
     assert bal_v1_swap.token_in_address == third_token_in_address

tests/test_traces.py

@@ -1,6 +1,6 @@
 from typing import List
 
-from mev_inspect.schemas.classified_traces import ClassifiedTrace
+from mev_inspect.schemas.traces import ClassifiedTrace
 from mev_inspect.traces import is_child_trace_address, get_child_traces
 
 from .helpers import make_many_unknown_traces

tests/utils.py

@@ -14,7 +14,7 @@ def load_test_block(block_number: int) -> Block:
 
     with open(block_path, "r") as block_file:
         block_json = json.load(block_file)
-        return Block(**block_json)
+        return Block(**block_json, block_timestamp=0)
 
 
 def load_comp_markets() -> Dict[str, str]: