Update README.md

graduate repo

.dockerignore

@@ -0,0 +1 @@
+cache

.env

@@ -1,8 +0,0 @@
-# Postgres
-POSTGRES_SERVER=db
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=password
-POSTGRES_DB=mev_inspect
-
-# SQLAlchemy
-SQLALCHEMY_DATABASE_URI=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_SERVER/$POSTGRES_DB

.github/workflows/github-actions.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Bootstrap poetry
         shell: bash
         run: |
-          curl -sL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py \
+          curl -sSL https://install.python-poetry.org \
             | python - -y 
 
       - name: Update PATH
@@ -51,8 +51,8 @@ jobs:
 
       - name: Run precommit
         run: |
-          poetry run pre-commit
+          poetry run pre-commit run --all-files
 
       - name: Test with pytest
         shell: bash
-        run: poetry run test
+        run: poetry run pytest --cov=mev_inspect tests

.gitignore

@@ -19,3 +19,12 @@ cache
 
 # k8s
 .helm
+
+# env
+.envrc
+
+# pycharm
+.idea
+
+.env
+.python-version

.pre-commit-config.yaml

@@ -1,9 +1,16 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: 20.8b1
+    rev: 22.3.0
     hooks:
-    - id: black
-      language_version: python3.9
+      - id: black
+        language_version: python3.9
+-   repo: local
+    hooks:
+      - id: isort
+        name: isort
+        entry: poetry run isort .
+        language: system
+        types: [python]
 -   repo: local
     hooks:
     - id: pylint
@@ -13,8 +20,9 @@ repos:
       language: system
       types: [python]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910
+    rev: v0.942
     hooks:
       - id: 'mypy'
         additional_dependencies:
           - 'pydantic'
+          - 'types-requests'

.pylintrc

@@ -433,7 +433,7 @@ int-import-graph=
 known-standard-library=
 
 # Force import order to recognize a module as part of a third party library.
-known-third-party=enchant
+known-third-party=alembic
 
 # Couples of modules and preferred modules, separated by a comma.
 preferred-modules=

CONTRIBUTING.md

@@ -0,0 +1,36 @@
+# Contributing guide
+
+Welcome to the Flashbots collective! We just ask you to be nice when you play with us.
+
+## Pre-commit
+
+We use pre-commit to maintain a consistent style, prevent errors, and ensure test coverage.
+
+To set up, install dependencies through `poetry`:
+
+```
+poetry install
+```
+
+Then install pre-commit hooks with:
+
+```
+poetry run pre-commit install
+```
+
+## Tests
+
+Run tests with:
+
+```
+./mev test
+```
+
+## Send a pull request
+
+- Your proposed changes should be first described and discussed in an issue.
+- Open the branch in a personal fork, not in the team repository.
+- Every pull request should be small and represent a single change. If the problem is complicated, split it in multiple issues and pull requests.
+- Every pull request should be covered by unit tests.
+
+We appreciate you, friend <3.

Dockerfile

@@ -1,21 +1,29 @@
-FROM python:3.9
+FROM python:3.9-slim-buster
 
-RUN pip install -U pip \
+ENV POETRY_VERSION=1.1.12
+
+RUN useradd --create-home flashbot \
     && apt-get update \
-    && curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
+    && apt-get install -y --no-install-recommends build-essential libffi-dev libpq-dev gcc procps \
+    && pip install poetry==$POETRY_VERSION \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-ENV PATH="${PATH}:/root/.poetry/bin"
+ENV PATH="${PATH}:/home/flashbot/.local/bin"
 
-COPY ./pyproject.toml /app/pyproject.toml
-COPY ./poetry.lock /app/poetry.lock
+COPY --chown=flashbot ./pyproject.toml /app/pyproject.toml
+COPY --chown=flashbot ./poetry.lock /app/poetry.lock
 WORKDIR /app/
 
-RUN poetry config virtualenvs.create false && \
-    poetry install
+USER flashbot
 
-COPY . /app
+RUN poetry config virtualenvs.create false \
+    && poetry install
+
+COPY --chown=flashbot . /app
 
 # easter eggs 😝
 RUN echo "PS1='🕵️:\[\033[1;36m\]\h \[\033[1;34m\]\W\[\033[0;35m\]\[\033[1;36m\]$ \[\033[0m\]'" >> ~/.bashrc
 
-CMD ["/bin/bash"]
+ENTRYPOINT [ "poetry" ]
+CMD [ "run", "python", "loop.py" ]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Flashbots
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MONOLITHIC.md

@@ -0,0 +1,126 @@
+
+# Running mev-inspect-py without kubernetes ('monolithic mode')
+
+Running mev-inspect-py outside of kubernetes can be useful for debug purposes. In this case, the steps for installation are:
+1. Install dependencies (pyenv, poetry, postgres)
+1. Set up python virtual environment using matching python version (3.9.x) and install required python modules using poetry
+1. Create postgres database
+1. Run database migrations
+
+The database credentials and archive node address used by mev-inspect-py need to be loaded into environment variables (both for database migrations and to run mev-inspect-py).
+
+## Ubuntu install instructions
+
+So, starting from a clean Ubuntu 22.04 installation, the prerequisites for pyenv, psycopg2 (python3-dev libpq-dev) can be installed with
+
+`sudo apt install -y make build-essential git libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev liblzma-dev python3-dev libpq-dev`
+
+### pyenv
+Install pyenv using the web installer
+
+`curl https://pyenv.run | bash`
+
+and add the following to `~/.bashrc` (if running locally) or `~/.profile` (if running over ssh).
+
+```
+export PYENV_ROOT="$HOME/.pyenv"
+command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"
+eval "$(pyenv init -)"
+```
+
+Then update the current shell by running `source ~/.bashrc` or `source ~/.profile` as appropriate.
+
+### Poetry
+
+Install Poetry using the web installer
+
+`curl -sSL https://install.python-poetry.org | python3 -`
+
+add the following to `~/.bashrc` (if running locally) or `~/.profile` (if running over ssh)
+
+`export PATH="/home/user/.local/bin:$PATH"`
+
+If running over ssh you should also add the following to `~/.profile` to prevent [Poetry errors](https://github.com/python-poetry/poetry/issues/1917) from a lack of active keyring:
+
+`export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring`
+
+Again update current shell by running `source ~/.bashrc` or `source ~/.profile` as appropriate.
+
+### postgres
+We have tested two alternatives for postgres - installing locally or as a container.
+
+#### Option 1: Installing locally
+
+To install locally from a clean Ubuntu 22.04 installation, run:
+`sudo apt install postgresql postgresql-contrib`
+
+Note: You may need to reconfigure your pg-hba.conf to allow local access.
+
+#### Option 2: Installing docker
+
+To avoid interfering with your local postgres instance, you may prefer to run postgres within a docker container.
+For docker installation instructions, please refer to https://docs.docker.com/engine/install/ubuntu/
+
+### mev-inspect-py
+
+With all dependencies now installed, clone the mev-inspec-py repo
+```
+git clone https://github.com/flashbots/mev-inspect-py.git
+cd mev-inspect-py
+```
+We now install the required pythn version and use Poetry to install the required python modules into a virtual environment. 
+
+```
+pyenv install 3.9.16
+pyenv local 3.9.16
+poetry env use 3.9.16
+poetry install
+```
+
+### Create database
+mev-inspect-py outputs to a postgres database, so we need to set this up. There are various ways of doing this, two options are presented here.
+
+#### Option 1 — Run postgres locally
+```
+sudo -u postgres psql
+\password
+postgres
+create database mev_inspect;
+\q
+```
+
+#### Option 2 — Use postgres docker image
+To avoid interfering with your local postgres instance, you may prefer to run postgres within a docker container. First ensure that postgres is not currently running to ensure port `5432` is available:
+`sudo systemctl stop postgresql`
+and then start a containerised postgres instance:
+`sudo docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=mev_inspect postgres`
+
+### Environment variables
+We will need to set a few environment variables to use mev-inspect-py. **These will be required every time mev-inspect-py runs**, so again you may wish to add these to your `~/.bashrc` and/or `~/.profile` as appropriate. Note that you need to substitute the correct URL for your archive node below if you are not running Erigon locally.
+```
+export POSTGRES_USER=postgres
+export POSTGRES_PASSWORD=postgres
+export POSTGRES_HOST=localhost
+export RPC_URL="http://127.0.0.1:8545"
+```
+### Database migrations
+Finally run the database migrations and fetch price information:
+
+```
+poetry run alembic upgrade head
+poetry run fetch-all-prices
+```
+
+## Usage instructions
+The same functionality available through kubernetes can be run in 'monolithic mode', but the relevant functions now need to be invoked by Poetry directly. So to inspect a single block, run for example:
+
+`poetry run inspect-block 16379706`
+
+Or to inspect a range of blocks:
+
+`poetry run inspect-many-blocks 16379606 16379706`
+
+Or to run the test suite:
+
+`poetry run pytest tests`
+

README.md

@@ -1,104 +1,306 @@
-# mev-inspect
-A [WIP] Ethereum MEV Inspector in Python managed by Poetry
+⚠️ This tool has been deprecated. You can visit [Flashbots Data](https://datasets.flashbots.net/) for historical mev-inspect data on Ethereum and join us on the [Flashbots forum](https://collective.flashbots.net). ⚠️
 
-## Containers
-mev-inspect's local setup is built on [Docker Compose](https://docs.docker.com/compose/)
+# mev-inspect-py
 
-By default it starts up:
-- `mev-inspect` - a container with the code in this repo used for running scripts
-- `db` - a postgres database instance
-- `pgadmin` - a postgres DB UI for querying and more (avaiable at localhost:5050)
+[![standard-readme compliant](https://img.shields.io/badge/readme%20style-standard-brightgreen.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme)
+[![Discord](https://img.shields.io/discord/755466764501909692)](https://discord.gg/7hvTycdNcK)
 
-## Running locally
-Setup [Docker](https://www.docker.com/products/docker-desktop)
-Setup [Poetry](https://python-poetry.org/docs/#osx--linux--bashonwindows-install-instructions)
+[Maximal extractable value](https://ethereum.org/en/developers/docs/mev/) inspector for Ethereum, to illuminate the [dark forest](https://www.paradigm.xyz/2020/08/ethereum-is-a-dark-forest/) 🌲💡
+
+Given a block, mev-inspect finds:
+- miner payments (gas + coinbase)
+- tokens transfers and profit
+- swaps and [arbitrages](https://twitter.com/bertcmiller/status/1427632028263059462)
+- ...and more
+
+Data is stored in Postgres for analysis.
+
+## Install
+
+mev-inspect-py is built to run on kubernetes locally and in production.
+
+### Dependencies
+
+- [docker](https://www.docker.com/products/docker-desktop)
+- [kind](https://kind.sigs.k8s.io/docs/user/quick-start), or a similar tool for running local Kubernetes clusters
+- [kubectl](https://kubernetes.io/docs/tasks/tools/)
+- [helm](https://helm.sh/docs/intro/install/)
+- [tilt](https://docs.tilt.dev/install.html)
+
+### Set up
+
+Create a new cluster with:
 
-Install dependencies through poetry
 ```
-poetry install
+kind create cluster
 ```
 
-Start the services (optionally as daemon)
-```
-poetry run start [-d]
-```
+Set an environment variable `RPC_URL` to an RPC for fetching blocks.
 
-Apply the latest migrations against the local DB:
-```
-poetry run exec alembic upgrade head
-``` 
+mev-inspect-py currently requires a node with support for Erigon traces and receipts (not geth yet 😔).
 
-Run inspect on a block
-```
-poetry run inspect -b/--block-number 11931270 -r/--rpc 'http://111.11.11.111:8545/'
-``` 
+[pokt.network](https://www.pokt.network/)'s "Ethereum Mainnet Archival with trace calls" is a good hosted option.
 
-To stop the services (if running in the background, otherwise just ctrl+c)
-```
-poetry run stop
-```
+Example:
 
-MEV container can be attached via
 ```
-poetry run attach
-```
-
-Running additional compose commands are possible through standard `docker
-compose ...` calls.  Check `docker compose help` for more tools available
-
-## Executing scripts
-Any script can be run from the mev-inspect container like
-```
-poetry run exec <your command here>
-```
-
-For example
-```
-poetry run exec python examples/uniswap_inspect.py -block_number=123 -rpc='111.111.111'
-```
-
-### Poetry Scripts
-```bash
-# code check
-poetry run lint # linting via Pylint
-poetry run test # testing and code coverage with Pytest
-poetry run isort # fixing imports 
-poetry run mypy # type checking 
-poetry run black # style guide 
-poetry run pre-commit run --all-files # runs Black, PyLint and MyPy
-# docker management
-poetry run start [-d] # starts all services, optionally as a daemon
-poetry run stop # shutsdown all services or just ctrl + c if foreground
-poetry run build # rebuilds containers
-poetry run attach # enters the mev-inspect container in interactive mode
-# launches inspection script
-poetry run inspect -b/--block-number 11931270 -r/--rpc 'http://111.11.11.111:8545/'
+export RPC_URL="http://111.111.111.111:8546"
 ```
 
 
-## Rebuilding containers
-After changes to the app's Dockerfile, rebuild with
+Next, start all services with:
+
 ```
-poetry run build
+tilt up
 ```
 
-## Using PGAdmin
+Press "space" to see a browser of the services starting up.
 
-1. Go to [localhost:5050](localhost:5050)
+On first startup, you'll need to apply database migrations with:
 
-2. Login with the PGAdmin username and password in `.env`
+```
+./mev exec alembic upgrade head
+```
 
-3. Add a new engine for mev_inspect with
-    - host: db
-    - user / password: see `.env`
+And load prices data
+
+```
+./mev prices fetch-all
+```
+
+## Monolithic (non-kubernetes) install instructions
+
+For an alternative means of running mev-inspect-py for smaller set-ups or debug purposes see the [monolithic install instructions](MONOLITHIC.md).
+
+## Usage
+
+### Inspect a single block
+
+Inspecting block [12914944](https://twitter.com/mevalphaleak/status/1420416437575901185):
+
+```
+./mev inspect 12914944
+```
+
+### Inspect many blocks
+
+Inspecting blocks 12914944 to 12914954:
+
+```
+./mev inspect-many 12914944 12914954
+```
+
+### Inspect all incoming blocks
+
+Start a block listener with:
+
+```
+./mev listener start
+```
+
+By default, it will pick up wherever you left off.
+If running for the first time, listener starts at the latest block.
+
+Tail logs for the listener with:
+
+```
+./mev listener tail
+```
+
+And stop the listener with:
+
+```
+./mev listener stop
+```
+
+### Backfilling
+
+For larger backfills, you can inspect many blocks in parallel
+
+To inspect blocks 12914944 to 12915044, run
+```
+./mev backfill 12914944 12915044
+```
+
+This queues the blocks in Redis to be pulled off by the mev-inspect-worker service
+
+To increase or decrease parallelism, update the replicaCount value for the mev-inspect-workers helm chart
+
+Locally, this can be done by editing Tiltfile and changing "replicaCount=1" to your desired parallelism:
+```
+k8s_yaml(helm(
+    './k8s/mev-inspect-workers',
+    name='mev-inspect-workers',
+    set=["replicaCount=1"],
+))
+```
+
+You can see worker pods spin up then complete by watching the status of all pods
+```
+watch kubectl get pods
+```
+
+To see progress and failed batches, connect to Redis with
+```
+./mev redis
+```
+
+For total messages, query:
+```
+HLEN dramatiq:default.msgs
+```
+
+For messages failed and waiting to retry in the delay queue (DQ), query:
+```
+HGETALL dramatiq:default.DQ.msgs
+```
+
+For messages permanently failed in the dead letter queue (XQ), query:
+```
+HGETALL dramatiq:default.XQ.msgs
+```
+
+To clear the queue, delete keys for the main queue and delay queue
+```
+DEL dramatiq:default.msgs
+DEL dramatiq:default.DQ.msgs
+```
+
+For more information on queues, see the [spec shared by dramatiq](https://github.com/Bogdanp/dramatiq/blob/24cbc0dc551797783f41b08ea461e1b5d23a4058/dramatiq/brokers/redis/dispatch.lua#L24-L43)
+
+**Backfilling a list of blocks**
+
+Create a file containing a block per row, for example blocks.txt containing:
+```
+12500000
+12500001
+12500002
+```
+
+Then queue the blocks with
+```
+cat blocks.txt | ./mev block-list
+```
+
+To watch the logs for a given worker pod, take its pod name using the above, then run:
+```
+kubectl logs -f pod/mev-inspect-worker-abcdefg
+```
+
+(where `mev-inspect-worker-abcdefg` is your actual pod name)
+
+
+### Exploring
+
+All inspect output data is stored in Postgres.
+
+To connect to the local Postgres database for querying, launch a client container with:
+
+```
+./mev db
+```
+
+When you see the prompt:
+
+```
+mev_inspect=#
+```
+
+You're ready to query!
+
+Try finding the total number of swaps decoded with UniswapV3Pool:
+
+```
+SELECT COUNT(*) FROM swaps WHERE abi_name='UniswapV3Pool';
+```
+
+or top 10 arbs by gross profit that took profit in WETH:
+
+```
+SELECT *
+FROM arbitrages
+WHERE profit_token_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'
+ORDER BY profit_amount DESC
+LIMIT 10;
+```
+
+Postgres tip: Enter `\x` to enter "Explanded display" mode which looks nicer for results with many columns.
+
+## FAQ
+
+### How do I delete / reset my local postgres data?
+
+Stop the system if running:
+
+```
+tilt down
+```
+
+Delete it with:
+
+```
+kubectl delete pvc data-postgresql-postgresql-0
+```
+
+Start back up again:
+
+```
+tilt up
+```
+
+And rerun migrations to create the tables again:
+
+```
+./mev exec alembic upgrade head
+```
+
+### I was using the docker-compose setup and want to switch to kube, now what?
+
+Re-add the old `docker-compose.yml` file to your mev-inspect-py directory.
+
+A copy can be found [here](https://github.com/flashbots/mev-inspect-py/blob/ef60c097719629a7d2dc56c6e6c9a100fb706f76/docker-compose.yml)
+
+Tear down docker-compose resources:
+
+```
+docker compose down
+```
+
+Then go through the steps in the current README for kube setup.
+
+### Error from server (AlreadyExists): pods "postgres-client" already exists
+
+This means the postgres client container didn't shut down correctly.
+
+Delete this one with:
+
+```
+kubectl delete pod/postgres-client
+```
+
+Then start it back up again.
+
+## Maintainers
+
+- [@lukevs](https://github.com/lukevs)
+- [@gheise](https://github.com/gheise)
+- [@bertmiller](https://github.com/bertmiller)
 
 ## Contributing
 
-Pre-commit is used to maintain a consistent style, prevent errors and ensure test coverage. 
+[Flashbots](https://flashbots.net) is a research and development collective working on mitigating the negative externalities of decentralized economies. We contribute with the larger free software community to illuminate the dark forest.
 
-Install pre-commit with:
-```
-poetry run pre-commit install
-```
+You are welcome here <3.
 
-Update README if needed
+- If you want to join us, come and say hi in our [Discord chat](https://discord.gg/7hvTycdNcK).
+- If you have a question, feedback or a bug report for this project, please [open a new Issue](https://github.com/flashbots/mev-inspect-py/issues).
+- If you would like to contribute with code, check the [CONTRIBUTING file](CONTRIBUTING.md).
+- We just ask you to be nice.
+
+## Security
+
+If you find a security vulnerability on this project or any other initiative related to Flashbots, please let us know sending an email to security@flashbots.net.
+
+---
+
+Made with ☀️  by the ⚡🤖 collective.

Tiltfile

@@ -1,22 +1,119 @@
-load('ext://helm_remote', 'helm_remote')
+load("ext://helm_remote", "helm_remote")
+load("ext://secret", "secret_from_dict")
+load("ext://configmap", "configmap_from_dict")
+
 helm_remote("postgresql",
-            repo_name='bitnami',
-            repo_url='https://charts.bitnami.com/bitnami',
-            set=["postgresqlPassword=password", "postgresqlDatabase=mev_inspect"],
+            repo_name="bitnami",
+            repo_url="https://charts.bitnami.com/bitnami",
+            set=["auth.postgresPassword=password", "auth.database=mev_inspect"],
 )
 
-load('ext://secret', 'secret_from_dict')
+helm_remote("redis",
+            repo_name="bitnami",
+            repo_url="https://charts.bitnami.com/bitnami",
+            set=["global.redis.password=password"],
+)
+
+k8s_yaml(configmap_from_dict("mev-inspect-rpc", inputs = {
+    "url" : os.environ["RPC_URL"],
+}))
+
+k8s_yaml(configmap_from_dict("mev-inspect-listener-healthcheck", inputs = {
+    "url" : os.getenv("LISTENER_HEALTHCHECK_URL", default=""),
+}))
+
 k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = {
     "username" : "postgres",
     "password": "password",
+    "host": "postgresql",
 }))
 
-docker_build('mev-inspect', '.',
+# if using https://github.com/taarushv/trace-db
+# k8s_yaml(secret_from_dict("trace-db-credentials", inputs = {
+#     "username" : "username",
+#     "password": "password",
+#     "host": "trace-db-postgresql",
+# }))
+
+docker_build("mev-inspect-py", ".",
     live_update=[
-        sync('.', '/app'),
-        run('cd /app && poetry install',
-            trigger='./pyproject.toml'),
+        sync(".", "/app"),
+        run("cd /app && poetry install",
+            trigger="./pyproject.toml"),
     ],
 )
 
-k8s_yaml("k8s/app.yaml")
+k8s_yaml(helm(
+    './k8s/mev-inspect',
+    name='mev-inspect',
+    set=[
+        "extraEnv[0].name=AWS_ACCESS_KEY_ID",
+        "extraEnv[0].value=foobar",
+        "extraEnv[1].name=AWS_SECRET_ACCESS_KEY",
+        "extraEnv[1].value=foobar",
+        "extraEnv[2].name=AWS_REGION",
+        "extraEnv[2].value=us-east-1",
+        "extraEnv[3].name=AWS_ENDPOINT_URL",
+        "extraEnv[3].value=http://localstack:4566",
+    ],
+))
+
+k8s_yaml(helm(
+    './k8s/mev-inspect-workers',
+    name='mev-inspect-workers',
+    set=[
+        "extraEnv[0].name=AWS_ACCESS_KEY_ID",
+        "extraEnv[0].value=foobar",
+        "extraEnv[1].name=AWS_SECRET_ACCESS_KEY",
+        "extraEnv[1].value=foobar",
+        "extraEnv[2].name=AWS_REGION",
+        "extraEnv[2].value=us-east-1",
+        "extraEnv[3].name=AWS_ENDPOINT_URL",
+        "extraEnv[3].value=http://localstack:4566",
+        "replicaCount=1",
+    ],
+))
+
+k8s_resource(
+    workload="mev-inspect",
+    resource_deps=["postgresql", "redis-master"],
+)
+
+k8s_resource(
+    workload="mev-inspect-workers",
+    resource_deps=["postgresql", "redis-master"],
+)
+
+# uncomment to enable price monitor
+# k8s_yaml(helm('./k8s/mev-inspect-prices', name='mev-inspect-prices'))
+# k8s_resource(workload="mev-inspect-prices", resource_deps=["postgresql"])
+
+local_resource(
+    'pg-port-forward',
+    serve_cmd='kubectl port-forward --namespace default svc/postgresql 5432:5432',
+    resource_deps=["postgresql"]
+)
+
+# if using local S3 exports
+#k8s_yaml(secret_from_dict("mev-inspect-export", inputs = {
+#    "export-bucket-name" : "local-export",
+#    "export-bucket-region": "us-east-1",
+#    "export-aws-access-key-id": "foobar",
+#    "export-aws-secret-access-key": "foobar",
+#}))
+
+#helm_remote(
+#    "localstack",
+#    repo_name="localstack-charts",
+#    repo_url="https://localstack.github.io/helm-charts",
+#)
+#
+#local_resource(
+#    'localstack-port-forward',
+#    serve_cmd='kubectl port-forward --namespace default svc/localstack 4566:4566',
+#    resource_deps=["localstack"]
+#)
+#
+#k8s_yaml(configmap_from_dict("mev-inspect-export", inputs = {
+#    "services": "s3",
+#}))

alembic/env.py

@@ -1,16 +1,14 @@
 from logging.config import fileConfig
 
-from sqlalchemy import engine_from_config
-from sqlalchemy import pool
-
 from alembic import context
+from sqlalchemy import engine_from_config, pool
 
-from mev_inspect.db import get_sqlalchemy_database_uri
+from mev_inspect.db import get_inspect_database_uri
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
-config.set_main_option("sqlalchemy.url", get_sqlalchemy_database_uri())
+config.set_main_option("sqlalchemy.url", get_inspect_database_uri())
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.

alembic/versions/04a3bb3740c3_change_miner_payments_and_transfers_.py

@@ -0,0 +1,54 @@
+"""Change miner payments and transfers primary keys to include block number
+
+Revision ID: 04a3bb3740c3
+Revises: a10d68643476
+Create Date: 2021-11-02 22:42:01.702538
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "04a3bb3740c3"
+down_revision = "a10d68643476"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # transfers
+    op.execute("ALTER TABLE transfers DROP CONSTRAINT transfers_pkey")
+    op.create_primary_key(
+        "transfers_pkey",
+        "transfers",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+    op.drop_index("ix_transfers_block_number")
+
+    # miner_payments
+    op.execute("ALTER TABLE miner_payments DROP CONSTRAINT miner_payments_pkey")
+    op.create_primary_key(
+        "miner_payments_pkey",
+        "miner_payments",
+        ["block_number", "transaction_hash"],
+    )
+    op.drop_index("ix_block_number")
+
+
+def downgrade():
+    # transfers
+    op.execute("ALTER TABLE transfers DROP CONSTRAINT transfers_pkey")
+    op.create_index("ix_transfers_block_number", "transfers", ["block_number"])
+    op.create_primary_key(
+        "transfers_pkey",
+        "transfers",
+        ["transaction_hash", "trace_address"],
+    )
+
+    # miner_payments
+    op.execute("ALTER TABLE miner_payments DROP CONSTRAINT miner_payments_pkey")
+    op.create_index("ix_block_number", "miner_payments", ["block_number"])
+    op.create_primary_key(
+        "miner_payments_pkey",
+        "miner_payments",
+        ["transaction_hash"],
+    )

alembic/versions/04b76ab1d2af_change_blocks_timestamp_to_timestamp.py

@@ -0,0 +1,35 @@
+"""Change blocks.timestamp to timestamp
+
+Revision ID: 04b76ab1d2af
+Revises: 2c90b2b8a80b
+Create Date: 2021-11-26 15:31:21.111693
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "04b76ab1d2af"
+down_revision = "0cef835f7b36"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column(
+        "blocks",
+        "block_timestamp",
+        type_=sa.TIMESTAMP,
+        nullable=False,
+        postgresql_using="TO_TIMESTAMP(block_timestamp)",
+    )
+
+
+def downgrade():
+    op.alter_column(
+        "blocks",
+        "block_timestamp",
+        type_=sa.Numeric,
+        nullable=False,
+        postgresql_using="extract(epoch FROM block_timestamp)",
+    )

alembic/versions/070819d86587_create_punk_snipe.py

@@ -0,0 +1,34 @@
+"""empty message
+
+Revision ID: 070819d86587
+Revises: d498bdb0a641
+Create Date: 2021-11-26 18:25:13.402822
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d498bdb0a641"
+down_revision = "b9fa1ecc9929"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "punk_snipes",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("trace_address", sa.String(256), nullable=False),
+        sa.Column("from_address", sa.String(256), nullable=False),
+        sa.Column("punk_index", sa.Numeric, nullable=False),
+        sa.Column("min_acceptance_price", sa.Numeric, nullable=False),
+        sa.Column("acceptance_price", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
+    )
+
+
+def downgrade():
+    op.drop_table("punk_snipes")

alembic/versions/083978d6e455_create_miner_payments_table.py

@@ -8,7 +8,6 @@ Create Date: 2021-08-30 17:42:25.548130
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "083978d6e455"
 down_revision = "92f28a2b4f52"

alembic/versions/0cef835f7b36_rename_pool_address_to_contract_address.py

@@ -0,0 +1,26 @@
+"""Rename pool_address to contract_address
+
+Revision ID: 0cef835f7b36
+Revises: 5427d62a2cc0
+Create Date: 2021-11-19 15:36:15.152622
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0cef835f7b36"
+down_revision = "5427d62a2cc0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column(
+        "swaps", "pool_address", nullable=False, new_column_name="contract_address"
+    )
+
+
+def downgrade():
+    op.alter_column(
+        "swaps", "contract_address", nullable=False, new_column_name="pool_address"
+    )

alembic/versions/15ba9c27ee8a_add_nullable_transaction_position_field_.py

@@ -0,0 +1,28 @@
+"""Add nullable transaction_position field to swaps and traces
+
+Revision ID: 15ba9c27ee8a
+Revises: 04b76ab1d2af
+Create Date: 2021-12-02 18:24:18.218880
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "15ba9c27ee8a"
+down_revision = "ead7eb8283b9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "classified_traces",
+        sa.Column("transaction_position", sa.Numeric, nullable=True),
+    )
+    op.add_column("swaps", sa.Column("transaction_position", sa.Numeric, nullable=True))
+
+
+def downgrade():
+    op.drop_column("classified_traces", "transaction_position")
+    op.drop_column("swaps", "transaction_position")

alembic/versions/205ce02374b3_add_received_collateral_address_to_liquidations.py

@@ -0,0 +1,26 @@
+"""Add received_collateral_address to liquidations
+
+Revision ID: 205ce02374b3
+Revises: c8363617aa07
+Create Date: 2021-10-04 19:52:40.017084
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "205ce02374b3"
+down_revision = "c8363617aa07"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "liquidations",
+        sa.Column("received_token_address", sa.String(256), nullable=True),
+    )
+
+
+def downgrade():
+    op.drop_column("liquidations", "received_token_address")

alembic/versions/2c90b2b8a80b_add_blocks_table.py

@@ -0,0 +1,28 @@
+"""Add blocks table
+
+Revision ID: 2c90b2b8a80b
+Revises: 04a3bb3740c3
+Create Date: 2021-11-17 18:29:13.065944
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "2c90b2b8a80b"
+down_revision = "04a3bb3740c3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "blocks",
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("block_timestamp", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("block_number"),
+    )
+
+
+def downgrade():
+    op.drop_table("blocks")

alembic/versions/320e56b0a99f_add_index_on_block_number_for_miner_.py

@@ -0,0 +1,22 @@
+"""Add index on block_number for miner_payments
+
+Revision ID: 320e56b0a99f
+Revises: a02f3f2c469f
+Create Date: 2021-09-14 11:11:41.559137
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "320e56b0a99f"
+down_revision = "a02f3f2c469f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_index("ix_block_number", "miner_payments", ["block_number"])
+
+
+def downgrade():
+    op.drop_index("ix_block_number", "miner_payments")

alembic/versions/3417f49d97b3_cahnge_swap_primary_key_to_include_.py

@@ -0,0 +1,45 @@
+"""Cahnge swap primary key to include block number
+
+Revision ID: 3417f49d97b3
+Revises: 205ce02374b3
+Create Date: 2021-11-02 20:50:32.854996
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "3417f49d97b3"
+down_revision = "205ce02374b3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("ALTER TABLE swaps DROP CONSTRAINT swaps_pkey CASCADE")
+    op.create_primary_key(
+        "swaps_pkey",
+        "swaps",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+    op.create_index(
+        "arbitrage_swaps_swaps_idx",
+        "arbitrage_swaps",
+        ["swap_transaction_hash", "swap_trace_address"],
+    )
+
+
+def downgrade():
+    op.drop_index("arbitrage_swaps_swaps_idx")
+    op.execute("ALTER TABLE swaps DROP CONSTRAINT swaps_pkey CASCADE")
+    op.create_primary_key(
+        "swaps_pkey",
+        "swaps",
+        ["transaction_hash", "trace_address"],
+    )
+    op.create_foreign_key(
+        "arbitrage_swaps_swaps_fkey",
+        "arbitrage_swaps",
+        "swaps",
+        ["swap_transaction_hash", "swap_trace_address"],
+        ["transaction_hash", "trace_address"],
+    )

alembic/versions/3c54832385e3_create_nft_trades_table.py

@@ -0,0 +1,40 @@
+"""Create NFT Trades table
+
+Revision ID: 3c54832385e3
+Revises: 4b9d289f2d74
+Create Date: 2021-12-19 22:50:28.936516
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "3c54832385e3"
+down_revision = "4b9d289f2d74"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "nft_trades",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("abi_name", sa.String(1024), nullable=False),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("transaction_position", sa.Numeric, nullable=False),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("trace_address", sa.String(256), nullable=False),
+        sa.Column("protocol", sa.String(256), nullable=False),
+        sa.Column("error", sa.String(256), nullable=True),
+        sa.Column("seller_address", sa.String(256), nullable=False),
+        sa.Column("buyer_address", sa.String(256), nullable=False),
+        sa.Column("payment_token_address", sa.String(256), nullable=False),
+        sa.Column("payment_amount", sa.Numeric, nullable=False),
+        sa.Column("collection_address", sa.String(256), nullable=False),
+        sa.Column("token_id", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("transaction_hash", "trace_address"),
+    )
+
+
+def downgrade():
+    op.drop_table("nft_trades")

alembic/versions/4b9d289f2d74_add_error_column_to_liquidations.py

@@ -0,0 +1,23 @@
+"""Add error column to liquidations
+
+Revision ID: 4b9d289f2d74
+Revises: 99d376cb93cc
+Create Date: 2021-12-23 14:54:28.406159
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4b9d289f2d74"
+down_revision = "99d376cb93cc"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("liquidations", sa.Column("error", sa.String(256), nullable=True))
+
+
+def downgrade():
+    op.drop_column("liquidations", "error")

alembic/versions/52d75a7e0533_add_punk_bid_acceptances.py

@@ -0,0 +1,33 @@
+"""empty message
+
+Revision ID: 52d75a7e0533
+Revises: 7cf0eeb41da0
+Create Date: 2021-11-26 20:35:58.954138
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "52d75a7e0533"
+down_revision = "7cf0eeb41da0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "punk_bid_acceptances",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("trace_address", sa.String(256), nullable=False),
+        sa.Column("from_address", sa.String(256), nullable=False),
+        sa.Column("punk_index", sa.Numeric, nullable=False),
+        sa.Column("min_price", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
+    )
+
+
+def downgrade():
+    op.drop_table("punk_bid_acceptances")

alembic/versions/5427d62a2cc0_change_transfers_trace_address_to_array.py

@@ -0,0 +1,46 @@
+"""Change transfers trace address to ARRAY
+
+Revision ID: 5427d62a2cc0
+Revises: d540242ae368
+Create Date: 2021-11-19 13:25:11.252774
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "5427d62a2cc0"
+down_revision = "d540242ae368"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_constraint("transfers_pkey", "transfers")
+    op.alter_column(
+        "transfers",
+        "trace_address",
+        type_=sa.ARRAY(sa.Integer),
+        nullable=False,
+        postgresql_using="trace_address::int[]",
+    )
+    op.create_primary_key(
+        "transfers_pkey",
+        "transfers",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+
+
+def downgrade():
+    op.drop_constraint("transfers_pkey", "transfers")
+    op.alter_column(
+        "transfers",
+        "trace_address",
+        type_=sa.String(256),
+        nullable=False,
+    )
+    op.create_primary_key(
+        "transfers_pkey",
+        "transfers",
+        ["block_number", "transaction_hash", "trace_address"],
+    )

alembic/versions/5c5375de15fd_add_block_number_to_nft_trades_primary_.py

@@ -0,0 +1,32 @@
+"""Add block_number to nft_trades primary key
+
+Revision ID: 5c5375de15fd
+Revises: e616420acd18
+Create Date: 2022-01-21 15:27:57.790340
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "5c5375de15fd"
+down_revision = "e616420acd18"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("ALTER TABLE nft_trades DROP CONSTRAINT nft_trades_pkey")
+    op.create_primary_key(
+        "nft_trades_pkey",
+        "nft_trades",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+
+
+def downgrade():
+    op.execute("ALTER TABLE nft_trades DROP CONSTRAINT nft_trades_pkey")
+    op.create_primary_key(
+        "nft_trades_pkey",
+        "nft_trades",
+        ["transaction_hash", "trace_address"],
+    )

alembic/versions/630783c18a93_make_gross_profit_nullable_on_summary.py

@@ -0,0 +1,22 @@
+"""Make gross profit nullable on summary
+
+Revision ID: 630783c18a93
+Revises: ab9a9e449ff9
+Create Date: 2022-01-19 23:09:51.816948
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "630783c18a93"
+down_revision = "ab9a9e449ff9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column("mev_summary", "gross_profit_usd", nullable=True)
+
+
+def downgrade():
+    op.alter_column("mev_summary", "gross_profit_usd", nullable=False)

alembic/versions/7cf0eeb41da0_add_punk_bids.py

@@ -0,0 +1,33 @@
+"""empty message
+
+Revision ID: 7cf0eeb41da0
+Revises: d498bdb0a641
+Create Date: 2021-11-26 20:27:28.936516
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "7cf0eeb41da0"
+down_revision = "d498bdb0a641"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "punk_bids",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("trace_address", sa.String(256), nullable=False),
+        sa.Column("from_address", sa.String(256), nullable=False),
+        sa.Column("punk_index", sa.Numeric, nullable=False),
+        sa.Column("price", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
+    )
+
+
+def downgrade():
+    op.drop_table("punk_bids")

alembic/versions/7eec417a4f3e_change_trace_addresses_to_array_types.py

@@ -8,7 +8,6 @@ Create Date: 2021-08-06 15:58:04.556762
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "7eec417a4f3e"
 down_revision = "9d8c69b3dccb"

alembic/versions/92f28a2b4f52_add_error_column_to_swaps.py

@@ -8,7 +8,6 @@ Create Date: 2021-08-17 03:46:21.498821
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "92f28a2b4f52"
 down_revision = "9b8ae51c5d56"

alembic/versions/99d376cb93cc_error.py

@@ -0,0 +1,23 @@
+"""error column
+
+Revision ID: 99d376cb93cc
+Revises: c4a7620a2d33
+Create Date: 2021-12-21 21:26:12.142484
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "99d376cb93cc"
+down_revision = "c4a7620a2d33"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("arbitrages", sa.Column("error", sa.String(256), nullable=True))
+
+
+def downgrade():
+    op.drop_column("arbitrages", "error")

alembic/versions/9b8ae51c5d56_add_swap_arbitrage_join_table.py

@@ -8,7 +8,6 @@ Create Date: 2021-08-06 17:06:55.364516
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "9b8ae51c5d56"
 down_revision = "7eec417a4f3e"

alembic/versions/9d8c69b3dccb_add_arbitrages_and_swap_join_table.py

@@ -8,7 +8,6 @@ Create Date: 2021-08-05 21:46:35.209199
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "9d8c69b3dccb"
 down_revision = "2116e2f36a19"

alembic/versions/a02f3f2c469f_create_latest_block_table.py

@@ -8,7 +8,6 @@ Create Date: 2021-09-13 21:32:27.181344
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "a02f3f2c469f"
 down_revision = "d70c08b4db6f"

alembic/versions/a10d68643476_change_classified_traces_primary_key_to_.py

@@ -0,0 +1,34 @@
+"""Change classified traces primary key to include block number
+
+Revision ID: a10d68643476
+Revises: 3417f49d97b3
+Create Date: 2021-11-02 22:03:26.312317
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a10d68643476"
+down_revision = "3417f49d97b3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("ALTER TABLE classified_traces DROP CONSTRAINT classified_traces_pkey")
+    op.create_primary_key(
+        "classified_traces_pkey",
+        "classified_traces",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+    op.drop_index("i_block_number")
+
+
+def downgrade():
+    op.execute("ALTER TABLE classified_traces DROP CONSTRAINT classified_traces_pkey")
+    op.create_index("i_block_number", "classified_traces", ["block_number"])
+    op.create_primary_key(
+        "classified_traces_pkey",
+        "classified_traces",
+        ["transaction_hash", "trace_address"],
+    )

alembic/versions/ab9a9e449ff9_create_mev_summary_table.py

@@ -0,0 +1,40 @@
+"""Create mev_summary table
+
+Revision ID: ab9a9e449ff9
+Revises: b26ab0051a88
+Create Date: 2022-01-18 18:36:42.865154
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "ab9a9e449ff9"
+down_revision = "b26ab0051a88"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "mev_summary",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("block_timestamp", sa.TIMESTAMP, nullable=False),
+        sa.Column("protocol", sa.String(256), nullable=True),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("type", sa.String(256), nullable=False),
+        sa.Column("gross_profit_usd", sa.Numeric, nullable=False),
+        sa.Column("miner_payment_usd", sa.Numeric, nullable=False),
+        sa.Column("gas_used", sa.Numeric, nullable=False),
+        sa.Column("gas_price", sa.Numeric, nullable=False),
+        sa.Column("coinbase_transfer", sa.Numeric, nullable=False),
+        sa.Column("gas_price_with_coinbase_transfer", sa.Numeric, nullable=False),
+        sa.Column("miner_address", sa.String(256), nullable=False),
+        sa.Column("base_fee_per_gas", sa.Numeric, nullable=False),
+        sa.Column("error", sa.String(256), nullable=True),
+    )
+
+
+def downgrade():
+    op.drop_table("mev_summary")

alembic/versions/b26ab0051a88_add_profit_amount_column_to_sandwiches_.py

@@ -0,0 +1,27 @@
+"""add profit_amount column to sandwiches table
+
+Revision ID: b26ab0051a88
+Revises: 3c54832385e3
+Create Date: 2022-01-16 13:45:10.190969
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "b26ab0051a88"
+down_revision = "3c54832385e3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "sandwiches", sa.Column("profit_token_address", sa.String(256), nullable=True)
+    )
+    op.add_column("sandwiches", sa.Column("profit_amount", sa.Numeric, nullable=True))
+
+
+def downgrade():
+    op.drop_column("sandwiches", "profit_token_address")
+    op.drop_column("sandwiches", "profit_amount")

alembic/versions/b9fa1ecc9929_remove_liq_column.py

@@ -0,0 +1,26 @@
+"""Remove collateral_token_address column
+
+Revision ID: b9fa1ecc9929
+Revises: 04b76ab1d2af
+Create Date: 2021-12-01 23:32:40.574108
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "b9fa1ecc9929"
+down_revision = "04b76ab1d2af"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_column("liquidations", "collateral_token_address")
+
+
+def downgrade():
+    op.add_column(
+        "liquidations",
+        sa.Column("collateral_token_address", sa.String(256), nullable=False),
+    )

alembic/versions/bba80d21c5a4_insert_tokens.py

@@ -0,0 +1,40 @@
+"""Add tokens to database
+
+Revision ID: bba80d21c5a4
+Revises: b26ab0051a88
+Create Date: 2022-01-19 22:19:59.514998
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "bba80d21c5a4"
+down_revision = "630783c18a93"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute(
+        """
+        INSERT INTO tokens (token_address,decimals) VALUES
+        ('0x514910771af9ca656af840dff83e8264ecf986ca',18),
+        ('0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',18),
+        ('0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',18),
+        ('0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e',18),
+        ('0x5d3a536e4d6dbd6114cc1ead35777bab948e3643',8),
+        ('0x2260fac5e5542a773aa44fbcfedf7c193bc2c599',8),
+        ('0x80fb784b7ed66730e8b1dbd9820afd29931aab03',18),
+        ('0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5',8),
+        ('0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48',6),
+        ('0xdac17f958d2ee523a2206206994597c13d831ec7',6),
+        ('0x6b175474e89094c44da98b954eedeac495271d0f',18),
+        ('0x0000000000085d4780b73119b644ae5ecd22b376',18),
+        ('0x39aa39c021dfbae8fac545936693ac917d5e7563',8),
+        ('0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9',18);
+        """
+    )
+
+
+def downgrade():
+    op.execute("DELETE FROM tokens")

alembic/versions/bdbb545f6c03_add_protocols_column_to_arbitrages.py

@@ -0,0 +1,26 @@
+"""Add protocols column to arbitrages
+
+Revision ID: bdbb545f6c03
+Revises: bba80d21c5a4
+Create Date: 2022-01-20 23:17:19.316008
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "bdbb545f6c03"
+down_revision = "bba80d21c5a4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "arbitrages",
+        sa.Column("protocols", sa.ARRAY(sa.String(256)), server_default="{}"),
+    )
+
+
+def downgrade():
+    op.drop_column("arbitrages", "protocols")

alembic/versions/c4a7620a2d33_create_tokens_table.py

@@ -0,0 +1,28 @@
+"""Create tokens table
+
+Revision ID: c4a7620a2d33
+Revises: 15ba9c27ee8a
+Create Date: 2021-12-21 19:12:33.940117
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "c4a7620a2d33"
+down_revision = "15ba9c27ee8a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "tokens",
+        sa.Column("token_address", sa.String(256), nullable=False),
+        sa.Column("decimals", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("token_address"),
+    )
+
+
+def downgrade():
+    op.drop_table("tokens")

alembic/versions/c5da44eb072c_add_index_for_classified_traces_block_.py

@@ -7,7 +7,6 @@ Create Date: 2021-07-30 17:37:27.335475
 """
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "c5da44eb072c"
 down_revision = "0660432b9840"

alembic/versions/c8363617aa07_create_liquidations_table.py

@@ -0,0 +1,37 @@
+"""Create liquidations table
+
+Revision ID: c8363617aa07
+Revises: cd96af55108e
+Create Date: 2021-09-29 14:00:06.857103
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "c8363617aa07"
+down_revision = "cd96af55108e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "liquidations",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("liquidated_user", sa.String(256), nullable=False),
+        sa.Column("liquidator_user", sa.String(256), nullable=False),
+        sa.Column("collateral_token_address", sa.String(256), nullable=False),
+        sa.Column("debt_token_address", sa.String(256), nullable=False),
+        sa.Column("debt_purchase_amount", sa.Numeric, nullable=False),
+        sa.Column("received_amount", sa.Numeric, nullable=False),
+        sa.Column("protocol", sa.String(256), nullable=True),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("trace_address", sa.String(256), nullable=False),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("transaction_hash", "trace_address"),
+    )
+
+
+def downgrade():
+    op.drop_table("liquidations")

alembic/versions/cd96af55108e_add_transfers_table.py

@@ -0,0 +1,38 @@
+"""Add transfers table
+
+Revision ID: cd96af55108e
+Revises: 5437dc68f4df
+Create Date: 2021-09-17 12:44:45.245137
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "cd96af55108e"
+down_revision = "320e56b0a99f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "transfers",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("trace_address", sa.String(256), nullable=False),
+        sa.Column("protocol", sa.String(256), nullable=True),
+        sa.Column("from_address", sa.String(256), nullable=False),
+        sa.Column("to_address", sa.String(256), nullable=False),
+        sa.Column("token_address", sa.String(256), nullable=False),
+        sa.Column("amount", sa.Numeric, nullable=False),
+        sa.Column("error", sa.String(256), nullable=True),
+        sa.PrimaryKeyConstraint("transaction_hash", "trace_address"),
+    )
+    op.create_index("ix_transfers_block_number", "transfers", ["block_number"])
+
+
+def downgrade():
+    op.drop_index("ix_transfers_block_number", "transfers")
+    op.drop_table("transfers")

alembic/versions/d540242ae368_create_usd_prices_table.py

@@ -0,0 +1,29 @@
+"""Create usd_prices table
+
+Revision ID: d540242ae368
+Revises: 2c90b2b8a80b
+Create Date: 2021-11-18 04:30:06.802857
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d540242ae368"
+down_revision = "2c90b2b8a80b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "prices",
+        sa.Column("timestamp", sa.TIMESTAMP),
+        sa.Column("usd_price", sa.Numeric, nullable=False),
+        sa.Column("token_address", sa.String(256), nullable=False),
+        sa.PrimaryKeyConstraint("token_address", "timestamp"),
+    )
+
+
+def downgrade():
+    op.drop_table("prices")

alembic/versions/d70c08b4db6f_add_to_address_and_from_address_to_.py

@@ -8,7 +8,6 @@ Create Date: 2021-08-30 22:10:04.186251
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "d70c08b4db6f"
 down_revision = "083978d6e455"

alembic/versions/e616420acd18_add_protocols_column_to_mev_summary.py

@@ -0,0 +1,26 @@
+"""Add protocols column to mev_summary
+
+Revision ID: e616420acd18
+Revises: bdbb545f6c03
+Create Date: 2022-01-21 00:11:51.516459
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "e616420acd18"
+down_revision = "bdbb545f6c03"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "mev_summary",
+        sa.Column("protocols", sa.ARRAY(sa.String(256)), server_default="{}"),
+    )
+
+
+def downgrade():
+    op.drop_column("mev_summary", "protocols")

alembic/versions/ead7eb8283b9_create_sandwiches_and_sandwiched_swaps_.py

@@ -0,0 +1,69 @@
+"""Create sandwiches and sandwiched swaps tables
+
+Revision ID: ead7eb8283b9
+Revises: a5d80460f0e6
+Create Date: 2021-12-03 16:37:28.077158
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "ead7eb8283b9"
+down_revision = "52d75a7e0533"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "sandwiches",
+        sa.Column("id", sa.String(256), primary_key=True),
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("sandwicher_address", sa.String(256), nullable=False),
+        sa.Column("frontrun_swap_transaction_hash", sa.String(256), nullable=False),
+        sa.Column("frontrun_swap_trace_address", sa.ARRAY(sa.Integer), nullable=False),
+        sa.Column("backrun_swap_transaction_hash", sa.String(256), nullable=False),
+        sa.Column("backrun_swap_trace_address", sa.ARRAY(sa.Integer), nullable=False),
+    )
+
+    op.create_index(
+        "ik_sandwiches_frontrun",
+        "sandwiches",
+        [
+            "block_number",
+            "frontrun_swap_transaction_hash",
+            "frontrun_swap_trace_address",
+        ],
+    )
+
+    op.create_index(
+        "ik_sandwiches_backrun",
+        "sandwiches",
+        ["block_number", "backrun_swap_transaction_hash", "backrun_swap_trace_address"],
+    )
+
+    op.create_table(
+        "sandwiched_swaps",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("sandwich_id", sa.String(1024), primary_key=True),
+        sa.Column("block_number", sa.Numeric, primary_key=True),
+        sa.Column("transaction_hash", sa.String(66), primary_key=True),
+        sa.Column("trace_address", sa.ARRAY(sa.Integer), primary_key=True),
+        sa.ForeignKeyConstraint(["sandwich_id"], ["sandwiches.id"], ondelete="CASCADE"),
+    )
+
+    op.create_index(
+        "ik_sandwiched_swaps_secondary",
+        "sandwiched_swaps",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+
+
+def downgrade():
+    op.drop_index("ik_sandwiched_swaps_secondary")
+    op.drop_table("sandwiched_swaps")
+    op.drop_index("ik_sandwiches_frontrun")
+    op.drop_index("ik_sandwiches_backrun")
+    op.drop_table("sandwiches")

cli.py

@@ -0,0 +1,219 @@
+import fileinput
+import logging
+import os
+import sys
+from datetime import datetime
+
+import click
+import dramatiq
+
+from mev_inspect.concurrency import coro
+from mev_inspect.crud.prices import write_prices
+from mev_inspect.db import get_inspect_session, get_trace_session
+from mev_inspect.inspector import MEVInspector
+from mev_inspect.prices import fetch_prices, fetch_prices_range
+from mev_inspect.queue.broker import connect_broker
+from mev_inspect.queue.tasks import (
+    LOW_PRIORITY,
+    LOW_PRIORITY_QUEUE,
+    backfill_export_task,
+    inspect_many_blocks_task,
+)
+from mev_inspect.s3_export import export_block
+
+RPC_URL_ENV = "RPC_URL"
+
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+@click.argument("block_number", type=int)
+@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
+@coro
+async def inspect_block_command(block_number: int, rpc: str):
+    inspect_db_session = get_inspect_session()
+    trace_db_session = get_trace_session()
+
+    inspector = MEVInspector(rpc)
+
+    await inspector.inspect_single_block(
+        inspect_db_session=inspect_db_session,
+        trace_db_session=trace_db_session,
+        block=block_number,
+    )
+
+
+@cli.command()
+@click.argument("block_number", type=int)
+@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
+@coro
+async def fetch_block_command(block_number: int, rpc: str):
+    trace_db_session = get_trace_session()
+
+    inspector = MEVInspector(rpc)
+    block = await inspector.create_from_block(
+        block_number=block_number,
+        trace_db_session=trace_db_session,
+    )
+
+    print(block.json())
+
+
+@cli.command()
+@click.argument("after_block", type=int)
+@click.argument("before_block", type=int)
+@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
+@click.option(
+    "--max-concurrency",
+    type=int,
+    help="maximum number of concurrent connections",
+    default=5,
+)
+@click.option(
+    "--request-timeout", type=int, help="timeout for requests to nodes", default=500
+)
+@coro
+async def inspect_many_blocks_command(
+    after_block: int,
+    before_block: int,
+    rpc: str,
+    max_concurrency: int,
+    request_timeout: int,
+):
+    inspect_db_session = get_inspect_session()
+    trace_db_session = get_trace_session()
+
+    inspector = MEVInspector(
+        rpc,
+        max_concurrency=max_concurrency,
+        request_timeout=request_timeout,
+    )
+    await inspector.inspect_many_blocks(
+        inspect_db_session=inspect_db_session,
+        trace_db_session=trace_db_session,
+        after_block=after_block,
+        before_block=before_block,
+    )
+
+
+@cli.command()
+def enqueue_block_list_command():
+    broker = connect_broker()
+    inspect_many_blocks_actor = dramatiq.actor(
+        inspect_many_blocks_task,
+        broker=broker,
+        queue_name=LOW_PRIORITY_QUEUE,
+        priority=LOW_PRIORITY,
+    )
+
+    for block_string in fileinput.input():
+        block = int(block_string)
+        logger.info(f"Sending {block} to {block+1}")
+        inspect_many_blocks_actor.send(block, block + 1)
+
+
+@cli.command()
+@click.argument("start_block", type=int)
+@click.argument("end_block", type=int)
+@click.argument("batch_size", type=int, default=10)
+def enqueue_many_blocks_command(start_block: int, end_block: int, batch_size: int):
+    broker = connect_broker()
+    inspect_many_blocks_actor = dramatiq.actor(
+        inspect_many_blocks_task,
+        broker=broker,
+        queue_name=LOW_PRIORITY_QUEUE,
+        priority=LOW_PRIORITY,
+    )
+
+    if start_block < end_block:
+        after_block = start_block
+        before_block = end_block
+
+        for batch_after_block in range(after_block, before_block, batch_size):
+            batch_before_block = min(batch_after_block + batch_size, before_block)
+            logger.info(f"Sending {batch_after_block} to {batch_before_block}")
+            inspect_many_blocks_actor.send(batch_after_block, batch_before_block)
+    else:
+        after_block = end_block
+        before_block = start_block
+
+        for batch_before_block in range(before_block, after_block, -1 * batch_size):
+            batch_after_block = max(batch_before_block - batch_size, after_block)
+            logger.info(f"Sending {batch_after_block} to {batch_before_block}")
+            inspect_many_blocks_actor.send(batch_after_block, batch_before_block)
+
+
+@cli.command()
+def fetch_all_prices():
+    inspect_db_session = get_inspect_session()
+
+    logger.info("Fetching prices")
+    prices = fetch_prices()
+
+    logger.info("Writing prices")
+    write_prices(inspect_db_session, prices)
+
+
+@cli.command()
+@click.argument("block_number", type=int)
+def enqueue_s3_export(block_number: int):
+    broker = connect_broker()
+    export_actor = dramatiq.actor(
+        backfill_export_task,
+        broker=broker,
+        queue_name=LOW_PRIORITY_QUEUE,
+        priority=LOW_PRIORITY,
+    )
+    logger.info(f"Sending block {block_number} export to queue")
+    export_actor.send(block_number)
+
+
+@cli.command()
+@click.argument("after_block", type=int)
+@click.argument("before_block", type=int)
+def enqueue_many_s3_exports(after_block: int, before_block: int):
+    broker = connect_broker()
+    export_actor = dramatiq.actor(
+        backfill_export_task,
+        broker=broker,
+        queue_name=LOW_PRIORITY_QUEUE,
+        priority=LOW_PRIORITY,
+    )
+    logger.info(f"Sending blocks {after_block} to {before_block} to queue")
+    for block_number in range(after_block, before_block):
+        export_actor.send(block_number)
+
+
+@cli.command()
+@click.argument("block_number", type=int)
+def s3_export(block_number: int):
+    inspect_db_session = get_inspect_session()
+    logger.info(f"Exporting {block_number}")
+    export_block(inspect_db_session, block_number)
+
+
+@cli.command()
+@click.argument("after", type=click.DateTime(formats=["%Y-%m-%d", "%m-%d-%Y"]))
+@click.argument("before", type=click.DateTime(formats=["%Y-%m-%d", "%m-%d-%Y"]))
+def fetch_range(after: datetime, before: datetime):
+    inspect_db_session = get_inspect_session()
+
+    logger.info("Fetching prices")
+    prices = fetch_prices_range(after, before)
+
+    logger.info("Writing prices")
+    write_prices(inspect_db_session, prices)
+
+
+def get_rpc_url() -> str:
+    return os.environ["RPC_URL"]
+
+
+if __name__ == "__main__":
+    cli()

docker-compose.yml

@@ -1,24 +0,0 @@
-services:
-  mev-inspect:
-    build: .
-    depends_on:
-      - db
-    env_file:
-      - .env
-    volumes:
-      - .:/app
-    tty: true
-
-  db:
-    image: postgres:12
-    volumes:
-      - mev-inspect-db-data:/var/lib/postgresql/data/pgdata
-    env_file:
-      - .env
-    environment:
-      - PGDATA=/var/lib/postgresql/data/pgdata
-    ports:
-      - 5432:5432
-
-volumes:
-  mev-inspect-db-data:

k8s/app.yaml

@@ -1,39 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: mev-inspect-deployment
-  labels:
-    app: mev-inspect
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: mev-inspect
-  template:
-    metadata:
-      labels:
-        app: mev-inspect
-    spec:
-      containers:
-      - name: mev-inspect
-        image: mev-inspect:latest
-        command: [ "/bin/bash", "-c", "--" ]
-        args: [ "while true; do sleep 30; done;" ]
-        env:
-        - name: POSTGRES_USER
-          valueFrom:
-            secretKeyRef:
-              name: mev-inspect-db-credentials
-              key: username
-        - name: POSTGRES_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: mev-inspect-db-credentials
-              key: password
-        livenessProbe:
-          exec:
-            command:
-            - ls
-            - /
-          initialDelaySeconds: 20
-          periodSeconds: 5

k8s/mev-inspect-prices/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

k8s/mev-inspect-prices/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: mev-inspect-prices
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

k8s/mev-inspect-prices/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "mev-inspect-prices.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mev-inspect-prices.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "mev-inspect-prices.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "mev-inspect-prices.labels" -}}
+helm.sh/chart: {{ include "mev-inspect-prices.chart" . }}
+{{ include "mev-inspect-prices.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "mev-inspect-prices.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "mev-inspect-prices.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "mev-inspect-prices.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "mev-inspect-prices.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

k8s/mev-inspect-prices/templates/cronjob.yaml

@@ -0,0 +1,35 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "mev-inspect-prices.fullname" . }}
+spec:
+  schedule: "0 */1 * * *"
+  successfulJobsHistoryLimit: 0
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - name: {{ .Chart.Name }}
+              image: "{{ .Values.image.repository }}"
+              imagePullPolicy: {{ .Values.image.pullPolicy }}
+              args:
+              - run
+              - fetch-all-prices
+              env:
+              - name: POSTGRES_HOST
+                valueFrom:
+                  secretKeyRef:
+                    name: mev-inspect-db-credentials
+                    key: host
+              - name: POSTGRES_USER
+                valueFrom:
+                  secretKeyRef:
+                    name: mev-inspect-db-credentials
+                    key: username
+              - name: POSTGRES_PASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: mev-inspect-db-credentials
+                    key: password
+          restartPolicy: Never

k8s/mev-inspect-prices/values.yaml

@@ -0,0 +1,7 @@
+image:
+  repository: mev-inspect-py
+  pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""

k8s/mev-inspect-workers/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

k8s/mev-inspect-workers/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: mev-inspect-workers
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

k8s/mev-inspect-workers/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "mev-inspect-worker.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mev-inspect-worker.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "mev-inspect-worker.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "mev-inspect-worker.labels" -}}
+helm.sh/chart: {{ include "mev-inspect-worker.chart" . }}
+{{ include "mev-inspect-worker.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "mev-inspect-worker.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "mev-inspect-worker.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "mev-inspect-worker.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "mev-inspect-worker.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

k8s/mev-inspect-workers/templates/deployment.yaml

@@ -0,0 +1,133 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "mev-inspect-worker.fullname" . }}
+  labels:
+    {{- include "mev-inspect-worker.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "mev-inspect-worker.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "mev-inspect-worker.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args: ["run", "dramatiq", "worker", "--threads=1", "--processes=1"]
+          livenessProbe:
+            exec:
+              command:
+              - ls
+              - /
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          env:
+          - name: POSTGRES_HOST
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: host
+          - name: POSTGRES_USER
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: username
+          - name: POSTGRES_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: password
+          - name: REDIS_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: redis
+                key: redis-password
+          - name: TRACE_DB_HOST
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: host
+                optional: true
+          - name: TRACE_DB_USER
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: username
+                optional: true
+          - name: TRACE_DB_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: password
+                optional: true
+          - name: RPC_URL
+            valueFrom:
+              configMapKeyRef:
+                name: mev-inspect-rpc
+                key: url
+          - name: LISTENER_HEALTHCHECK_URL
+            valueFrom:
+              configMapKeyRef:
+                name: mev-inspect-listener-healthcheck
+                key: url
+                optional: true
+          - name: EXPORT_BUCKET_NAME
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-export
+                key: export-bucket-name
+                optional: true
+          - name: EXPORT_BUCKET_REGION
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-export
+                key: export-bucket-region
+                optional: true
+          - name: EXPORT_AWS_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-export
+                key: export-aws-access-key-id
+                optional: true
+          - name: EXPORT_AWS_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-export
+                key: export-aws-secret-access-key
+                optional: true
+          {{- range .Values.extraEnv }}
+          - name: {{ .name }}
+            value: {{ .value }}
+          {{- end }} 
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

k8s/mev-inspect-workers/values.yaml

@@ -0,0 +1,45 @@
+# Default values for mev-inspect-workers
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+  repository: mev-inspect-py:latest
+  pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+    - ALL
+  # readOnlyRootFilesystem: true
+  runAsNonRoot: true
+  runAsUser: 1000
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

k8s/mev-inspect/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

k8s/mev-inspect/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: mev-inspect
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

k8s/mev-inspect/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "mev-inspect.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mev-inspect.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "mev-inspect.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "mev-inspect.labels" -}}
+helm.sh/chart: {{ include "mev-inspect.chart" . }}
+{{ include "mev-inspect.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "mev-inspect.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "mev-inspect.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "mev-inspect.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "mev-inspect.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

k8s/mev-inspect/templates/deployment.yaml

@@ -0,0 +1,133 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "mev-inspect.fullname" . }}
+  labels:
+    {{- include "mev-inspect.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "mev-inspect.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "mev-inspect.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args: ["run", "python", "loop.py"]
+          livenessProbe:
+            exec:
+              command:
+              - ls
+              - /
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          env:
+          - name: POSTGRES_HOST
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: host
+          - name: POSTGRES_USER
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: username
+          - name: POSTGRES_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: password
+          - name: REDIS_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: redis
+                key: redis-password
+          - name: TRACE_DB_HOST
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: host
+                optional: true
+          - name: TRACE_DB_USER
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: username
+                optional: true
+          - name: TRACE_DB_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: password
+                optional: true
+          - name: RPC_URL
+            valueFrom:
+              configMapKeyRef:
+                name: mev-inspect-rpc
+                key: url
+          - name: LISTENER_HEALTHCHECK_URL
+            valueFrom:
+              configMapKeyRef:
+                name: mev-inspect-listener-healthcheck
+                key: url
+                optional: true
+          - name: EXPORT_BUCKET_NAME
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-export
+                key: export-bucket-name
+                optional: true
+          - name: EXPORT_BUCKET_REGION
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-export
+                key: export-bucket-region
+                optional: true
+          - name: EXPORT_AWS_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-export
+                key: export-aws-access-key-id
+                optional: true
+          - name: EXPORT_AWS_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-export
+                key: export-aws-secret-access-key
+                optional: true
+          {{- range .Values.extraEnv }}
+          - name: {{ .name }}
+            value: {{ .value }}
+          {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

k8s/mev-inspect/values.yaml

@@ -0,0 +1,46 @@
+# Default values for mev-inspect.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+  repository: mev-inspect-py:latest
+  pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+      - all
+  #readOnlyRootFilesystem: true
+  runAsNonRoot: true
+  runAsUser: 1000
+
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

listener

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+set -e
+
+NAME=listener
+PIDFILE=/home/flashbot/$NAME.pid
+DAEMON=/bin/bash
+DAEMON_OPTS='-c "poetry run python listener.py"'
+
+case "$1" in
+  start)
+        echo -n "Starting daemon: "$NAME
+	start-stop-daemon \
+        --background \
+        --chdir /app \
+        --chuid flashbot \
+        --start \
+        --quiet \
+        --pidfile $PIDFILE \
+        --make-pidfile \
+        --startas /bin/bash -- -c "poetry run python listener.py"
+        echo "."
+	;;
+  stop)
+        echo -n "Stopping daemon: "$NAME
+	start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE
+    rm $PIDFILE
+        echo "."
+	;;
+  tail)
+        tail -f listener.log
+	;;
+  restart)
+        echo -n "Restarting daemon: "$NAME
+	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile $PIDFILE
+    rm $PIDFILE
+	start-stop-daemon \
+        --background \
+        --chdir /app \
+        --chuid flashbot \
+        --start \
+        --quiet \
+        --pidfile $PIDFILE \
+        --make-pidfile \
+        --startas /bin/bash -- -c "poetry run python listener.py"
+        echo "."
+    ;;
+
+  *)
+	echo "Usage: "$1" {start|stop|restart|tail}"
+	exit 1
+esac
+
+exit 0

listener.py

@@ -0,0 +1,126 @@
+import asyncio
+import logging
+import os
+
+import dramatiq
+from aiohttp_retry import ExponentialRetry, RetryClient
+
+from mev_inspect.block import get_latest_block_number
+from mev_inspect.concurrency import coro
+from mev_inspect.crud.latest_block_update import (
+    find_latest_block_update,
+    update_latest_block,
+)
+from mev_inspect.db import get_inspect_session, get_trace_session
+from mev_inspect.inspector import MEVInspector
+from mev_inspect.provider import get_base_provider
+from mev_inspect.queue.broker import connect_broker
+from mev_inspect.queue.tasks import (
+    HIGH_PRIORITY,
+    HIGH_PRIORITY_QUEUE,
+    realtime_export_task,
+)
+from mev_inspect.signal_handler import GracefulKiller
+
+logging.basicConfig(filename="listener.log", filemode="a", level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# lag to make sure the blocks we see are settled
+BLOCK_NUMBER_LAG = 5
+
+
+@coro
+async def run():
+    rpc = os.getenv("RPC_URL")
+    if rpc is None:
+        raise RuntimeError("Missing environment variable RPC_URL")
+
+    healthcheck_url = os.getenv("LISTENER_HEALTHCHECK_URL")
+
+    logger.info("Starting...")
+
+    killer = GracefulKiller()
+
+    inspect_db_session = get_inspect_session()
+    trace_db_session = get_trace_session()
+
+    broker = connect_broker()
+    export_actor = dramatiq.actor(
+        realtime_export_task,
+        broker=broker,
+        queue_name=HIGH_PRIORITY_QUEUE,
+        priority=HIGH_PRIORITY,
+    )
+
+    inspector = MEVInspector(rpc)
+    base_provider = get_base_provider(rpc)
+
+    while not killer.kill_now:
+        await inspect_next_block(
+            inspector,
+            inspect_db_session,
+            trace_db_session,
+            base_provider,
+            healthcheck_url,
+            export_actor,
+        )
+
+    logger.info("Stopping...")
+
+
+async def inspect_next_block(
+    inspector: MEVInspector,
+    inspect_db_session,
+    trace_db_session,
+    base_provider,
+    healthcheck_url,
+    export_actor,
+):
+
+    latest_block_number = await get_latest_block_number(base_provider)
+    last_written_block = find_latest_block_update(inspect_db_session)
+
+    logger.info(f"Latest block: {latest_block_number}")
+    logger.info(f"Last written block: {last_written_block}")
+
+    if last_written_block is None:
+        # maintain lag if no blocks written yet
+        last_written_block = latest_block_number - BLOCK_NUMBER_LAG - 1
+
+    if last_written_block < (latest_block_number - BLOCK_NUMBER_LAG):
+        block_number = last_written_block + 1
+
+        logger.info(f"Writing block: {block_number}")
+
+        await inspector.inspect_single_block(
+            inspect_db_session=inspect_db_session,
+            trace_db_session=trace_db_session,
+            block=block_number,
+        )
+
+        update_latest_block(inspect_db_session, block_number)
+
+        logger.info(f"Sending block {block_number} for export")
+        export_actor.send(block_number)
+
+        if healthcheck_url:
+            await ping_healthcheck_url(healthcheck_url)
+    else:
+        await asyncio.sleep(5)
+
+
+async def ping_healthcheck_url(url):
+    retry_options = ExponentialRetry(attempts=3)
+
+    async with RetryClient(
+        raise_for_status=False, retry_options=retry_options
+    ) as client:
+        async with client.get(url) as _response:
+            pass
+
+
+if __name__ == "__main__":
+    try:
+        run()
+    except Exception as e:
+        logger.error(e)

loop.py

@@ -0,0 +1,21 @@
+import logging
+import time
+
+from mev_inspect.signal_handler import GracefulKiller
+
+logging.basicConfig(filename="loop.log", level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def run():
+    logger.info("Starting...")
+
+    killer = GracefulKiller()
+    while not killer.kill_now:
+        time.sleep(1)
+
+    logger.info("Stopping...")
+
+
+if __name__ == "__main__":
+    run()

mev

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+
+set -e
+
+DB_NAME=mev_inspect
+
+function get_kube_secret(){
+    kubectl get secrets $1 -o jsonpath="{.data.$2}" | base64 --decode
+}
+
+function get_kube_db_secret(){
+    kubectl get secrets mev-inspect-db-credentials -o jsonpath="{.data.$1}" | base64 --decode
+}
+
+function db(){
+    host=$(get_kube_secret "mev-inspect-db-credentials" "host")
+    username=$(get_kube_secret "mev-inspect-db-credentials" "username")
+    password=$(get_kube_secret "mev-inspect-db-credentials" "password")
+
+    kubectl run -i --rm --tty postgres-client-$RANDOM \
+        --env="PGPASSWORD=$password"  \
+        --image=jbergknoff/postgresql-client \
+        -- $DB_NAME --host=$host --user=$username
+}
+
+function redis(){
+    echo "To continue, enter 'shift + r'"
+    redis_password=$(get_kube_secret "redis" "redis-password")
+    kubectl run -i --rm --tty \
+        --namespace default redis-client-$RANDOM \
+        --env REDIS_PASSWORD=$redis_password \
+        --image docker.io/bitnami/redis:6.2.6-debian-10-r0 \
+        --command -- redis-cli -h redis-master -a $redis_password
+}
+
+case "$1" in
+  db)
+        echo "Connecting to $DB_NAME"
+        db
+	;;
+  redis)
+        echo "Connecting to redis"
+        redis
+    ;;
+  listener)
+        kubectl exec -ti deploy/mev-inspect -- ./listener $2
+	;;
+  block-list)
+        echo "Backfilling blocks from stdin"
+        kubectl exec -i deploy/mev-inspect -- poetry run enqueue-block-list
+	;;
+  backfill)
+        after_block_number=$2
+        before_block_number=$3
+
+        echo "Backfilling from $after_block_number to $before_block_number"
+        kubectl exec -ti deploy/mev-inspect -- poetry run enqueue-many-blocks $after_block_number $before_block_number
+	;;
+  inspect)
+        block_number=$2
+        echo "Inspecting block $block_number"
+        kubectl exec -ti deploy/mev-inspect -- poetry run inspect-block $block_number
+	;;
+  inspect-many)
+        after_block_number=$2
+        before_block_number=$3
+        echo "Inspecting from block $after_block_number to $before_block_number"
+        kubectl exec -ti deploy/mev-inspect -- \
+            poetry run inspect-many-blocks $after_block_number $before_block_number
+	;;
+  test)
+        shift
+        echo "Running tests"
+        kubectl exec -ti deploy/mev-inspect -- poetry run pytest tests $@
+	;;
+  fetch)
+      block_number=$2
+      echo "Fetching block $block_number"
+      kubectl exec -ti deploy/mev-inspect -- poetry run fetch-block $block_number
+  ;;
+  prices)
+        shift
+        case "$1" in
+          fetch-all)
+                echo "Running price fetch-all"
+                kubectl exec -ti deploy/mev-inspect -- \
+                    poetry run fetch-all-prices
+            ;;
+          fetch-range)
+                after=$2
+                before=$3
+                echo "Running price fetch-range"
+                kubectl exec -ti deploy/mev-inspect -- \
+                    poetry run fetch-range $after $before
+            ;;
+            *)
+            echo "prices usage: "$1" {fetch-all}"
+            exit 1
+        esac
+  ;;
+  backfill-export)
+        after_block=$2
+        before_block=$3
+
+        echo "Sending $after_block to $before_block export to queue"
+        kubectl exec -ti deploy/mev-inspect -- poetry run enqueue-many-s3-exports $after_block $before_block
+  ;;
+  enqueue-s3-export)
+        block_number=$2
+
+        echo "Sending $block_number export to queue"
+        kubectl exec -ti deploy/mev-inspect -- poetry run enqueue-s3-export $block_number
+  ;;
+  s3-export)
+        block_number=$2
+
+        echo "Exporting $block_number"
+        kubectl exec -ti deploy/mev-inspect -- poetry run s3-export $block_number
+  ;;
+  exec)
+        shift
+        kubectl exec -ti deploy/mev-inspect -- $@
+  ;;
+  *)
+	echo "Usage: "$1" {db|backfill|inspect|test}"
+	exit 1
+esac
+
+exit 0

mev_inspect/abi.py

@@ -4,23 +4,39 @@ from typing import Optional
 
 from pydantic import parse_obj_as
 
-from mev_inspect.schemas import ABI
-from mev_inspect.schemas.classified_traces import Protocol
-
+from mev_inspect.schemas.abi import ABI
+from mev_inspect.schemas.traces import Protocol
 
 THIS_FILE_DIRECTORY = Path(__file__).parents[0]
 ABI_DIRECTORY_PATH = THIS_FILE_DIRECTORY / "abis"
 
 
-def get_abi(abi_name: str, protocol: Optional[Protocol]) -> Optional[ABI]:
+def get_abi_path(abi_name: str, protocol: Optional[Protocol]) -> Optional[Path]:
     abi_filename = f"{abi_name}.json"
     abi_path = (
         ABI_DIRECTORY_PATH / abi_filename
         if protocol is None
         else ABI_DIRECTORY_PATH / protocol.value / abi_filename
     )
-
     if abi_path.is_file():
+        return abi_path
+
+    return None
+
+
+# raw abi, for instantiating contract for queries (as opposed to classification, see below)
+def get_raw_abi(abi_name: str, protocol: Optional[Protocol]) -> Optional[str]:
+    abi_path = get_abi_path(abi_name, protocol)
+    if abi_path is not None:
+        with abi_path.open() as abi_file:
+            return abi_file.read()
+
+    return None
+
+
+def get_abi(abi_name: str, protocol: Optional[Protocol]) -> Optional[ABI]:
+    abi_path = get_abi_path(abi_name, protocol)
+    if abi_path is not None:
         with abi_path.open() as abi_file:
             abi_json = json.load(abi_file)
             return parse_obj_as(ABI, abi_json)

mev_inspect/abis/aave/aTokens.json

@@ -0,0 +1,615 @@
+[
+  {
+    "anonymous": false,
+    "inputs": [
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "owner",
+        "type": "address"
+      },
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "spender",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      }
+    ],
+    "name": "Approval",
+    "type": "event"
+  },
+  {
+    "anonymous": false,
+    "inputs": [
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "from",
+        "type": "address"
+      },
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "to",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "index",
+        "type": "uint256"
+      }
+    ],
+    "name": "BalanceTransfer",
+    "type": "event"
+  },
+  {
+    "anonymous": false,
+    "inputs": [
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "from",
+        "type": "address"
+      },
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "target",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "index",
+        "type": "uint256"
+      }
+    ],
+    "name": "Burn",
+    "type": "event"
+  },
+  {
+    "anonymous": false,
+    "inputs": [
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "underlyingAsset",
+        "type": "address"
+      },
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "pool",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "address",
+        "name": "treasury",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "address",
+        "name": "incentivesController",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint8",
+        "name": "aTokenDecimals",
+        "type": "uint8"
+      },
+      {
+        "indexed": false,
+        "internalType": "string",
+        "name": "aTokenName",
+        "type": "string"
+      },
+      {
+        "indexed": false,
+        "internalType": "string",
+        "name": "aTokenSymbol",
+        "type": "string"
+      },
+      {
+        "indexed": false,
+        "internalType": "bytes",
+        "name": "params",
+        "type": "bytes"
+      }
+    ],
+    "name": "Initialized",
+    "type": "event"
+  },
+  {
+    "anonymous": false,
+    "inputs": [
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "from",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "index",
+        "type": "uint256"
+      }
+    ],
+    "name": "Mint",
+    "type": "event"
+  },
+  {
+    "anonymous": false,
+    "inputs": [
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "from",
+        "type": "address"
+      },
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "to",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      }
+    ],
+    "name": "Transfer",
+    "type": "event"
+  },
+  {
+    "inputs": [
+      
+    ],
+    "name": "UNDERLYING_ASSET_ADDRESS",
+    "outputs": [
+      {
+        "internalType": "address",
+        "name": "",
+        "type": "address"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "owner",
+        "type": "address"
+      },
+      {
+        "internalType": "address",
+        "name": "spender",
+        "type": "address"
+      }
+    ],
+    "name": "allowance",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "spender",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      }
+    ],
+    "name": "approve",
+    "outputs": [
+      {
+        "internalType": "bool",
+        "name": "",
+        "type": "bool"
+      }
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "account",
+        "type": "address"
+      }
+    ],
+    "name": "balanceOf",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "user",
+        "type": "address"
+      },
+      {
+        "internalType": "address",
+        "name": "receiverOfUnderlying",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      },
+      {
+        "internalType": "uint256",
+        "name": "index",
+        "type": "uint256"
+      }
+    ],
+    "name": "burn",
+    "outputs": [
+      
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      
+    ],
+    "name": "getIncentivesController",
+    "outputs": [
+      {
+        "internalType": "contract IAaveIncentivesController",
+        "name": "",
+        "type": "address"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "user",
+        "type": "address"
+      }
+    ],
+    "name": "getScaledUserBalanceAndSupply",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      },
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "user",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      }
+    ],
+    "name": "handleRepayment",
+    "outputs": [
+      
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "contract ILendingPool",
+        "name": "pool",
+        "type": "address"
+      },
+      {
+        "internalType": "address",
+        "name": "treasury",
+        "type": "address"
+      },
+      {
+        "internalType": "address",
+        "name": "underlyingAsset",
+        "type": "address"
+      },
+      {
+        "internalType": "contract IAaveIncentivesController",
+        "name": "incentivesController",
+        "type": "address"
+      },
+      {
+        "internalType": "uint8",
+        "name": "aTokenDecimals",
+        "type": "uint8"
+      },
+      {
+        "internalType": "string",
+        "name": "aTokenName",
+        "type": "string"
+      },
+      {
+        "internalType": "string",
+        "name": "aTokenSymbol",
+        "type": "string"
+      },
+      {
+        "internalType": "bytes",
+        "name": "params",
+        "type": "bytes"
+      }
+    ],
+    "name": "initialize",
+    "outputs": [
+      
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "user",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      },
+      {
+        "internalType": "uint256",
+        "name": "index",
+        "type": "uint256"
+      }
+    ],
+    "name": "mint",
+    "outputs": [
+      {
+        "internalType": "bool",
+        "name": "",
+        "type": "bool"
+      }
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      },
+      {
+        "internalType": "uint256",
+        "name": "index",
+        "type": "uint256"
+      }
+    ],
+    "name": "mintToTreasury",
+    "outputs": [
+      
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "user",
+        "type": "address"
+      }
+    ],
+    "name": "scaledBalanceOf",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      
+    ],
+    "name": "scaledTotalSupply",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      
+    ],
+    "name": "totalSupply",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "recipient",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      }
+    ],
+    "name": "transfer",
+    "outputs": [
+      {
+        "internalType": "bool",
+        "name": "",
+        "type": "bool"
+      }
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "sender",
+        "type": "address"
+      },
+      {
+        "internalType": "address",
+        "name": "recipient",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      }
+    ],
+    "name": "transferFrom",
+    "outputs": [
+      {
+        "internalType": "bool",
+        "name": "",
+        "type": "bool"
+      }
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "from",
+        "type": "address"
+      },
+      {
+        "internalType": "address",
+        "name": "to",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      }
+    ],
+    "name": "transferOnLiquidation",
+    "outputs": [
+      
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "user",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      }
+    ],
+    "name": "transferUnderlyingTo",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  }
+]

mev_inspect/arbitrages.py

@@ -1,8 +1,11 @@
 from itertools import groupby
-from typing import List, Optional
+from typing import List, Optional, Tuple
 
 from mev_inspect.schemas.arbitrages import Arbitrage
 from mev_inspect.schemas.swaps import Swap
+from mev_inspect.utils import equal_within_percent
+
+MAX_TOKEN_AMOUNT_PERCENT_DIFFERENCE = 0.01
 
 
 def get_arbitrages(swaps: List[Swap]) -> List[Arbitrage]:
@@ -23,70 +26,168 @@ def get_arbitrages(swaps: List[Swap]) -> List[Arbitrage]:
 
 
 def _get_arbitrages_from_swaps(swaps: List[Swap]) -> List[Arbitrage]:
-    pool_addresses = {swap.pool_address for swap in swaps}
+    """
+    An arbitrage is defined as multiple swaps in a series that result in the initial token being returned
+    to the initial sender address.
+
+    There are 2 types of swaps that are most common (99%+).
+    Case I (fully routed):
+    BOT -> A/B -> B/C -> C/A -> BOT
+
+    Case II (always return to bot):
+    BOT -> A/B -> BOT -> B/C -> BOT -> A/C -> BOT
+
+    There is only 1 correct way to route Case I, but for Case II the following valid routes could be found:
+    A->B->C->A / B->C->A->B / C->A->B->C. Thus when multiple valid routes are found we filter to the set that
+    happen in valid order.
+    """
 
     all_arbitrages = []
 
-    for index, first_swap in enumerate(swaps):
-        other_swaps = swaps[:index] + swaps[index + 1 :]
+    start_ends = _get_all_start_end_swaps(swaps)
+    if len(start_ends) == 0:
+        return []
 
-        if first_swap.from_address not in pool_addresses:
-            arbitrage = _get_arbitrage_starting_with_swap(first_swap, other_swaps)
+    used_swaps: List[Swap] = []
 
-            if arbitrage is not None:
-                all_arbitrages.append(arbitrage)
+    for (start, ends) in start_ends:
+        if start in used_swaps:
+            continue
 
-    return all_arbitrages
+        unused_ends = [end for end in ends if end not in used_swaps]
+        route = _get_shortest_route(start, unused_ends, swaps)
 
-
-def _get_arbitrage_starting_with_swap(
-    start_swap: Swap,
-    other_swaps: List[Swap],
-) -> Optional[Arbitrage]:
-    swap_path = [start_swap]
-    current_swap: Swap = start_swap
-
-    while True:
-        next_swap = _get_swap_from_address(
-            current_swap.to_address,
-            current_swap.token_out_address,
-            other_swaps,
-        )
-
-        if next_swap is None:
-            return None
-
-        swap_path.append(next_swap)
-        current_swap = next_swap
-
-        if (
-            current_swap.to_address == start_swap.from_address
-            and current_swap.token_out_address == start_swap.token_in_address
-        ):
-
-            start_amount = start_swap.token_in_amount
-            end_amount = current_swap.token_out_amount
+        if route is not None:
+            start_amount = route[0].token_in_amount
+            end_amount = route[-1].token_out_amount
             profit_amount = end_amount - start_amount
+            error = None
+            for swap in route:
+                if swap.error is not None:
+                    error = swap.error
 
-            return Arbitrage(
-                swaps=swap_path,
-                block_number=start_swap.block_number,
-                transaction_hash=start_swap.transaction_hash,
-                account_address=start_swap.from_address,
-                profit_token_address=start_swap.token_in_address,
+            arb = Arbitrage(
+                swaps=route,
+                block_number=route[0].block_number,
+                transaction_hash=route[0].transaction_hash,
+                account_address=route[0].from_address,
+                profit_token_address=route[0].token_in_address,
                 start_amount=start_amount,
                 end_amount=end_amount,
                 profit_amount=profit_amount,
+                error=error,
             )
 
-    return None
+            all_arbitrages.append(arb)
+            used_swaps.extend(route)
+
+    if len(all_arbitrages) == 1:
+        return all_arbitrages
+    else:
+        return [
+            arb
+            for arb in all_arbitrages
+            if (arb.swaps[0].trace_address < arb.swaps[-1].trace_address)
+        ]
 
 
-def _get_swap_from_address(
-    address: str, token_address: str, swaps: List[Swap]
-) -> Optional[Swap]:
-    for swap in swaps:
-        if swap.pool_address == address and swap.token_in_address == token_address:
-            return swap
+def _get_shortest_route(
+    start_swap: Swap,
+    end_swaps: List[Swap],
+    all_swaps: List[Swap],
+    max_route_length: Optional[int] = None,
+) -> Optional[List[Swap]]:
+    if len(end_swaps) == 0:
+        return None
 
-    return None
+    if max_route_length is not None and max_route_length < 2:
+        return None
+
+    for end_swap in end_swaps:
+        if _swap_outs_match_swap_ins(start_swap, end_swap):
+            return [start_swap, end_swap]
+
+    if max_route_length is not None and max_route_length == 2:
+        return None
+
+    other_swaps = [
+        swap for swap in all_swaps if (swap is not start_swap and swap not in end_swaps)
+    ]
+
+    if len(other_swaps) == 0:
+        return None
+
+    shortest_remaining_route = None
+    max_remaining_route_length = (
+        None if max_route_length is None else max_route_length - 1
+    )
+
+    for next_swap in other_swaps:
+        if _swap_outs_match_swap_ins(start_swap, next_swap):
+            shortest_from_next = _get_shortest_route(
+                next_swap,
+                end_swaps,
+                other_swaps,
+                max_route_length=max_remaining_route_length,
+            )
+
+            if shortest_from_next is not None and (
+                shortest_remaining_route is None
+                or len(shortest_from_next) < len(shortest_remaining_route)
+            ):
+                shortest_remaining_route = shortest_from_next
+                max_remaining_route_length = len(shortest_from_next) - 1
+
+    if shortest_remaining_route is None:
+        return None
+    else:
+        return [start_swap] + shortest_remaining_route
+
+
+def _get_all_start_end_swaps(swaps: List[Swap]) -> List[Tuple[Swap, List[Swap]]]:
+    """
+    Gets the set of all possible openings and corresponding closing swaps for an arbitrage via
+    - swap[start].token_in == swap[end].token_out
+    - swap[start].from_address == swap[end].to_address
+    - not swap[start].from_address in all_pool_addresses
+    - not swap[end].to_address in all_pool_addresses
+    """
+    pool_addrs = [swap.contract_address for swap in swaps]
+    valid_start_ends: List[Tuple[Swap, List[Swap]]] = []
+
+    for index, potential_start_swap in enumerate(swaps):
+        ends_for_start: List[Swap] = []
+        remaining_swaps = swaps[:index] + swaps[index + 1 :]
+
+        for potential_end_swap in remaining_swaps:
+            if (
+                potential_start_swap.token_in_address
+                == potential_end_swap.token_out_address
+                and potential_start_swap.contract_address
+                != potential_end_swap.contract_address
+                and potential_start_swap.from_address == potential_end_swap.to_address
+                and not potential_start_swap.from_address in pool_addrs
+            ):
+
+                ends_for_start.append(potential_end_swap)
+
+        if len(ends_for_start) > 0:
+            valid_start_ends.append((potential_start_swap, ends_for_start))
+
+    return valid_start_ends
+
+
+def _swap_outs_match_swap_ins(swap_out, swap_in) -> bool:
+    return (
+        swap_out.token_out_address == swap_in.token_in_address
+        and (
+            swap_out.contract_address == swap_in.from_address
+            or swap_out.to_address == swap_in.contract_address
+            or swap_out.to_address == swap_in.from_address
+        )
+        and equal_within_percent(
+            swap_out.token_out_amount,
+            swap_in.token_in_amount,
+            MAX_TOKEN_AMOUNT_PERCENT_DIFFERENCE,
+        )
+    )

mev_inspect/block.py

@@ -1,58 +1,208 @@
-from pathlib import Path
-from typing import List
+import asyncio
+import logging
+from typing import List, Optional
 
+from sqlalchemy import orm
 from web3 import Web3
 
 from mev_inspect.fees import fetch_base_fee_per_gas
-from mev_inspect.schemas import Block, Trace, TraceType
+from mev_inspect.schemas.blocks import Block
 from mev_inspect.schemas.receipts import Receipt
+from mev_inspect.schemas.traces import Trace, TraceType
+from mev_inspect.utils import hex_to_int
+
+logger = logging.getLogger(__name__)
 
 
-cache_directory = "./cache"
+async def get_latest_block_number(base_provider) -> int:
+    latest_block = await base_provider.make_request(
+        "eth_getBlockByNumber",
+        ["latest", False],
+    )
+
+    return hex_to_int(latest_block["result"]["number"])
 
 
-def create_from_block_number(
-    base_provider, w3: Web3, block_number: int, should_cache: bool
+async def create_from_block_number(
+    w3: Web3,
+    block_number: int,
+    trace_db_session: Optional[orm.Session],
 ) -> Block:
-    if not should_cache:
-        return fetch_block(w3, base_provider, block_number)
-
-    cache_path = _get_cache_path(block_number)
-
-    if cache_path.is_file():
-        print(f"Cache for block {block_number} exists, " "loading data from cache")
-
-        return Block.parse_file(cache_path)
-    else:
-        print(f"Cache for block {block_number} did not exist, getting data")
-
-        block = fetch_block(w3, base_provider, block_number)
-
-        cache_block(cache_path, block)
-
-        return block
-
-
-def fetch_block(w3, base_provider, block_number: int) -> Block:
-    block_json = w3.eth.get_block(block_number)
-    receipts_json = base_provider.make_request("eth_getBlockReceipts", [block_number])
-    traces_json = w3.parity.trace_block(block_number)
-
-    receipts: List[Receipt] = [
-        Receipt(**receipt) for receipt in receipts_json["result"]
-    ]
-    traces = [Trace(**trace_json) for trace_json in traces_json]
-    base_fee_per_gas = fetch_base_fee_per_gas(w3, block_number)
+    block_timestamp, receipts, traces, base_fee_per_gas = await asyncio.gather(
+        _find_or_fetch_block_timestamp(w3, block_number, trace_db_session),
+        _find_or_fetch_block_receipts(w3, block_number, trace_db_session),
+        _find_or_fetch_block_traces(w3, block_number, trace_db_session),
+        _find_or_fetch_base_fee_per_gas(w3, block_number, trace_db_session),
+    )
+    miner_address = await _find_or_fetch_miner_address(w3, block_number, traces)
 
     return Block(
         block_number=block_number,
-        miner=block_json["miner"],
+        block_timestamp=block_timestamp,
+        miner=miner_address,
         base_fee_per_gas=base_fee_per_gas,
         traces=traces,
         receipts=receipts,
     )
 
 
+async def _find_or_fetch_block_timestamp(
+    w3,
+    block_number: int,
+    trace_db_session: Optional[orm.Session],
+) -> int:
+    if trace_db_session is not None:
+        existing_block_timestamp = _find_block_timestamp(trace_db_session, block_number)
+        if existing_block_timestamp is not None:
+            return existing_block_timestamp
+
+    return await _fetch_block_timestamp(w3, block_number)
+
+
+async def _find_or_fetch_block_receipts(
+    w3,
+    block_number: int,
+    trace_db_session: Optional[orm.Session],
+) -> List[Receipt]:
+    if trace_db_session is not None:
+        existing_block_receipts = _find_block_receipts(trace_db_session, block_number)
+        if existing_block_receipts is not None:
+            return existing_block_receipts
+
+    return await _fetch_block_receipts(w3, block_number)
+
+
+async def _find_or_fetch_block_traces(
+    w3,
+    block_number: int,
+    trace_db_session: Optional[orm.Session],
+) -> List[Trace]:
+    if trace_db_session is not None:
+        existing_block_traces = _find_block_traces(trace_db_session, block_number)
+        if existing_block_traces is not None:
+            return existing_block_traces
+
+    return await _fetch_block_traces(w3, block_number)
+
+
+async def _find_or_fetch_base_fee_per_gas(
+    w3,
+    block_number: int,
+    trace_db_session: Optional[orm.Session],
+) -> int:
+    if trace_db_session is not None:
+        existing_base_fee_per_gas = _find_base_fee_per_gas(
+            trace_db_session, block_number
+        )
+        if existing_base_fee_per_gas is not None:
+            return existing_base_fee_per_gas
+
+    return await fetch_base_fee_per_gas(w3, block_number)
+
+
+async def _fetch_block_timestamp(w3, block_number: int) -> int:
+    block_json = await w3.eth.get_block(block_number)
+    return block_json["timestamp"]
+
+
+async def _fetch_block_receipts(w3, block_number: int) -> List[Receipt]:
+    receipts_json = await w3.eth.get_block_receipts(block_number)
+    return [Receipt(**receipt) for receipt in receipts_json]
+
+
+async def _fetch_block_traces(w3, block_number: int) -> List[Trace]:
+    traces_json = await w3.eth.trace_block(block_number)
+    return [Trace(**trace_json) for trace_json in traces_json]
+
+
+def _find_block_timestamp(
+    trace_db_session: orm.Session,
+    block_number: int,
+) -> Optional[int]:
+    result = trace_db_session.execute(
+        "SELECT block_timestamp FROM block_timestamps WHERE block_number = :block_number",
+        params={"block_number": block_number},
+    ).one_or_none()
+
+    if result is None:
+        return None
+    else:
+        (block_timestamp,) = result
+        return block_timestamp
+
+
+def _find_block_traces(
+    trace_db_session: orm.Session,
+    block_number: int,
+) -> Optional[List[Trace]]:
+    result = trace_db_session.execute(
+        "SELECT raw_traces FROM block_traces WHERE block_number = :block_number",
+        params={"block_number": block_number},
+    ).one_or_none()
+
+    if result is None:
+        return None
+    else:
+        (traces_json,) = result
+        return [Trace(**trace_json) for trace_json in traces_json]
+
+
+def _find_block_receipts(
+    trace_db_session: orm.Session,
+    block_number: int,
+) -> Optional[List[Receipt]]:
+    result = trace_db_session.execute(
+        "SELECT raw_receipts FROM block_receipts WHERE block_number = :block_number",
+        params={"block_number": block_number},
+    ).one_or_none()
+
+    if result is None:
+        return None
+    else:
+        (receipts_json,) = result
+        return [Receipt(**receipt) for receipt in receipts_json]
+
+
+def _find_base_fee_per_gas(
+    trace_db_session: orm.Session,
+    block_number: int,
+) -> Optional[int]:
+    result = trace_db_session.execute(
+        "SELECT base_fee_in_wei FROM base_fee WHERE block_number = :block_number",
+        params={"block_number": block_number},
+    ).one_or_none()
+
+    if result is None:
+        return None
+    else:
+        (base_fee,) = result
+        return base_fee
+
+
+async def _find_or_fetch_miner_address(
+    w3,
+    block_number: int,
+    traces: List[Trace],
+) -> Optional[str]:
+    # eth1 blocks
+    miner_address = _get_miner_address_from_traces(traces)
+    if miner_address is not None:
+        return miner_address
+    return await _fetch_miner_eth2(w3, block_number)
+
+
+async def _fetch_miner_eth2(w3, block_number: int) -> Optional[str]:
+    block_json = await w3.eth.get_block(block_number)
+    return block_json["miner"]
+
+
+def _get_miner_address_from_traces(traces: List[Trace]) -> Optional[str]:
+    for trace in traces:
+        if trace.type == TraceType.reward:
+            return trace.action["author"]
+    return None
+
+
 def get_transaction_hashes(calls: List[Trace]) -> List[str]:
     result = []
 
@@ -65,15 +215,3 @@ def get_transaction_hashes(calls: List[Trace]) -> List[str]:
                 result.append(call.transaction_hash)
 
     return result
-
-
-def cache_block(cache_path: Path, block: Block):
-    write_mode = "w" if cache_path.is_file() else "x"
-
-    with open(cache_path, mode=write_mode) as cache_file:
-        cache_file.write(block.json())
-
-
-def _get_cache_path(block_number: int) -> Path:
-    cache_directory_path = Path(cache_directory)
-    return cache_directory_path / f"{block_number}-new.json"

mev_inspect/classifiers/helpers.py

@@ -0,0 +1,208 @@
+from typing import List, Optional, Sequence
+
+from mev_inspect.schemas.nft_trades import NftTrade
+from mev_inspect.schemas.prices import ETH_TOKEN_ADDRESS
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import ClassifiedTrace, DecodedCallTrace
+from mev_inspect.schemas.transfers import Transfer
+
+
+def create_nft_trade_from_transfers(
+    trace: DecodedCallTrace,
+    child_transfers: List[Transfer],
+    collection_address: str,
+    seller_address: str,
+    buyer_address: str,
+    exchange_wallet_address: str,
+) -> Optional[NftTrade]:
+    transfers_to_buyer = _filter_transfers(child_transfers, to_address=buyer_address)
+    transfers_to_seller = _filter_transfers(child_transfers, to_address=seller_address)
+
+    if len(transfers_to_buyer) != 1 or len(transfers_to_seller) != 1:
+        return None
+
+    if transfers_to_buyer[0].token_address != collection_address:
+        return None
+
+    payment_token_address = transfers_to_seller[0].token_address
+    payment_amount = transfers_to_seller[0].amount
+    token_id = transfers_to_buyer[0].amount
+
+    transfers_from_seller_to_exchange = _filter_transfers(
+        child_transfers,
+        from_address=seller_address,
+        to_address=exchange_wallet_address,
+    )
+    transfers_from_buyer_to_exchange = _filter_transfers(
+        child_transfers,
+        from_address=buyer_address,
+        to_address=exchange_wallet_address,
+    )
+    for fee in [
+        *transfers_from_seller_to_exchange,
+        *transfers_from_buyer_to_exchange,
+    ]:
+        # Assumes that exchange fees are paid with the same token as the sale
+        payment_amount -= fee.amount
+
+    return NftTrade(
+        abi_name=trace.abi_name,
+        transaction_hash=trace.transaction_hash,
+        transaction_position=trace.transaction_position,
+        block_number=trace.block_number,
+        trace_address=trace.trace_address,
+        protocol=trace.protocol,
+        error=trace.error,
+        seller_address=seller_address,
+        buyer_address=buyer_address,
+        payment_token_address=payment_token_address,
+        payment_amount=payment_amount,
+        collection_address=collection_address,
+        token_id=token_id,
+    )
+
+
+def create_swap_from_pool_transfers(
+    trace: DecodedCallTrace,
+    recipient_address: str,
+    prior_transfers: List[Transfer],
+    child_transfers: List[Transfer],
+) -> Optional[Swap]:
+    pool_address = trace.to_address
+
+    transfers_to_pool = []
+
+    if trace.value is not None and trace.value > 0:
+        transfers_to_pool = [_build_eth_transfer(trace)]
+
+    if len(transfers_to_pool) == 0:
+        transfers_to_pool = _filter_transfers(prior_transfers, to_address=pool_address)
+
+    if len(transfers_to_pool) == 0:
+        transfers_to_pool = _filter_transfers(child_transfers, to_address=pool_address)
+
+    if len(transfers_to_pool) == 0:
+        return None
+
+    transfers_from_pool_to_recipient = _filter_transfers(
+        child_transfers, to_address=recipient_address, from_address=pool_address
+    )
+
+    if len(transfers_from_pool_to_recipient) != 1:
+        return None
+
+    transfer_in = transfers_to_pool[-1]
+    transfer_out = transfers_from_pool_to_recipient[0]
+
+    if transfer_in.token_address == transfer_out.token_address:
+        return None
+
+    return Swap(
+        abi_name=trace.abi_name,
+        transaction_hash=trace.transaction_hash,
+        transaction_position=trace.transaction_position,
+        block_number=trace.block_number,
+        trace_address=trace.trace_address,
+        contract_address=pool_address,
+        protocol=trace.protocol,
+        from_address=transfer_in.from_address,
+        to_address=transfer_out.to_address,
+        token_in_address=transfer_in.token_address,
+        token_in_amount=transfer_in.amount,
+        token_out_address=transfer_out.token_address,
+        token_out_amount=transfer_out.amount,
+        error=trace.error,
+    )
+
+
+def create_swap_from_recipient_transfers(
+    trace: DecodedCallTrace,
+    pool_address: str,
+    recipient_address: str,
+    prior_transfers: List[Transfer],
+    child_transfers: List[Transfer],
+) -> Optional[Swap]:
+    transfers_from_recipient = _filter_transfers(
+        [*prior_transfers, *child_transfers], from_address=recipient_address
+    )
+    transfers_to_recipient = _filter_transfers(
+        child_transfers, to_address=recipient_address
+    )
+
+    if len(transfers_from_recipient) != 1 or len(transfers_to_recipient) != 1:
+        return None
+
+    transfer_in = transfers_from_recipient[0]
+    transfer_out = transfers_to_recipient[0]
+
+    return Swap(
+        abi_name=trace.abi_name,
+        transaction_hash=trace.transaction_hash,
+        transaction_position=trace.transaction_position,
+        block_number=trace.block_number,
+        trace_address=trace.trace_address,
+        contract_address=pool_address,
+        protocol=trace.protocol,
+        from_address=transfer_in.from_address,
+        to_address=transfer_out.to_address,
+        token_in_address=transfer_in.token_address,
+        token_in_amount=transfer_in.amount,
+        token_out_address=transfer_out.token_address,
+        token_out_amount=transfer_out.amount,
+        error=trace.error,
+    )
+
+
+def _build_eth_transfer(trace: ClassifiedTrace) -> Transfer:
+    return Transfer(
+        block_number=trace.block_number,
+        transaction_hash=trace.transaction_hash,
+        trace_address=trace.trace_address,
+        amount=trace.value,
+        to_address=trace.to_address,
+        from_address=trace.from_address,
+        token_address=ETH_TOKEN_ADDRESS,
+    )
+
+
+def _filter_transfers(
+    transfers: Sequence[Transfer],
+    to_address: Optional[str] = None,
+    from_address: Optional[str] = None,
+) -> List[Transfer]:
+    filtered_transfers = []
+
+    for transfer in transfers:
+        if to_address is not None and transfer.to_address != to_address:
+            continue
+
+        if from_address is not None and transfer.from_address != from_address:
+            continue
+
+        filtered_transfers.append(transfer)
+
+    return filtered_transfers
+
+
+def get_received_transfer(
+    liquidator: str, child_transfers: List[Transfer]
+) -> Optional[Transfer]:
+    """Get transfer from AAVE to liquidator"""
+
+    for transfer in child_transfers:
+        if transfer.to_address == liquidator:
+            return transfer
+
+    return None
+
+
+def get_debt_transfer(
+    liquidator: str, child_transfers: List[Transfer]
+) -> Optional[Transfer]:
+    """Get transfer from liquidator to AAVE"""
+
+    for transfer in child_transfers:
+        if transfer.from_address == liquidator:
+            return transfer
+
+    return None

mev_inspect/classifiers/specs/__init__.py

@@ -1,11 +1,21 @@
+from typing import Dict, Optional, Tuple, Type
+
+from mev_inspect.schemas.classifiers import Classifier, ClassifierSpec
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
+
 from .aave import AAVE_CLASSIFIER_SPECS
+from .balancer import BALANCER_CLASSIFIER_SPECS
+from .bancor import BANCOR_CLASSIFIER_SPECS
+from .compound import COMPOUND_CLASSIFIER_SPECS
+from .cream import CREAM_CLASSIFIER_SPECS
+from .cryptopunks import CRYPTOPUNKS_CLASSIFIER_SPECS
 from .curve import CURVE_CLASSIFIER_SPECS
 from .erc20 import ERC20_CLASSIFIER_SPECS
+from .opensea import OPENSEA_CLASSIFIER_SPECS
 from .uniswap import UNISWAP_CLASSIFIER_SPECS
 from .weth import WETH_CLASSIFIER_SPECS
 from .zero_ex import ZEROX_CLASSIFIER_SPECS
 
-
 ALL_CLASSIFIER_SPECS = (
     ERC20_CLASSIFIER_SPECS
     + WETH_CLASSIFIER_SPECS
@@ -13,4 +23,26 @@ ALL_CLASSIFIER_SPECS = (
     + UNISWAP_CLASSIFIER_SPECS
     + AAVE_CLASSIFIER_SPECS
     + ZEROX_CLASSIFIER_SPECS
+    + BALANCER_CLASSIFIER_SPECS
+    + COMPOUND_CLASSIFIER_SPECS
+    + CREAM_CLASSIFIER_SPECS
+    + CRYPTOPUNKS_CLASSIFIER_SPECS
+    + OPENSEA_CLASSIFIER_SPECS
+    + BANCOR_CLASSIFIER_SPECS
 )
+
+_SPECS_BY_ABI_NAME_AND_PROTOCOL: Dict[
+    Tuple[str, Optional[Protocol]], ClassifierSpec
+] = {(spec.abi_name, spec.protocol): spec for spec in ALL_CLASSIFIER_SPECS}
+
+
+def get_classifier(
+    trace: DecodedCallTrace,
+) -> Optional[Type[Classifier]]:
+    abi_name_and_protocol = (trace.abi_name, trace.protocol)
+    spec = _SPECS_BY_ABI_NAME_AND_PROTOCOL.get(abi_name_and_protocol)
+
+    if spec is not None:
+        return spec.classifiers.get(trace.function_signature)
+
+    return None

mev_inspect/classifiers/specs/aave.py

@@ -1,15 +1,93 @@
-from mev_inspect.schemas.classified_traces import (
-    Classification,
+from typing import List, Optional
+
+from mev_inspect.classifiers.helpers import get_debt_transfer, get_received_transfer
+from mev_inspect.schemas.classifiers import (
+    ClassifiedTrace,
     ClassifierSpec,
-    Protocol,
+    DecodedCallTrace,
+    LiquidationClassifier,
+    TransferClassifier,
 )
+from mev_inspect.schemas.liquidations import Liquidation
+from mev_inspect.schemas.traces import Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+
+class AaveLiquidationClassifier(LiquidationClassifier):
+    @staticmethod
+    def parse_liquidation(
+        liquidation_trace: DecodedCallTrace,
+        child_transfers: List[Transfer],
+        child_traces: List[ClassifiedTrace],
+    ) -> Optional[Liquidation]:
+
+        liquidator = liquidation_trace.from_address
+        liquidated = liquidation_trace.inputs["_user"]
+
+        debt_token_address = liquidation_trace.inputs["_reserve"]
+        received_token_address = liquidation_trace.inputs["_collateral"]
+
+        debt_purchase_amount = None
+        received_amount = None
+
+        debt_transfer = get_debt_transfer(liquidator, child_transfers)
+
+        received_transfer = get_received_transfer(liquidator, child_transfers)
+
+        if debt_transfer is not None and received_transfer is not None:
+
+            debt_token_address = debt_transfer.token_address
+            debt_purchase_amount = debt_transfer.amount
+
+            received_token_address = received_transfer.token_address
+            received_amount = received_transfer.amount
+
+            return Liquidation(
+                liquidated_user=liquidated,
+                debt_token_address=debt_token_address,
+                liquidator_user=liquidator,
+                debt_purchase_amount=debt_purchase_amount,
+                protocol=Protocol.aave,
+                received_amount=received_amount,
+                received_token_address=received_token_address,
+                transaction_hash=liquidation_trace.transaction_hash,
+                trace_address=liquidation_trace.trace_address,
+                block_number=liquidation_trace.block_number,
+                error=liquidation_trace.error,
+            )
+
+        else:
+            return None
+
+
+class AaveTransferClassifier(TransferClassifier):
+    @staticmethod
+    def get_transfer(trace: DecodedCallTrace) -> Transfer:
+        return Transfer(
+            block_number=trace.block_number,
+            transaction_hash=trace.transaction_hash,
+            trace_address=trace.trace_address,
+            amount=trace.inputs["value"],
+            to_address=trace.inputs["to"],
+            from_address=trace.inputs["from"],
+            token_address=trace.to_address,
+        )
+
 
 AAVE_SPEC = ClassifierSpec(
     abi_name="AaveLendingPool",
     protocol=Protocol.aave,
-    classifications={
-        "liquidationCall(address,address,address,uint256,bool)": Classification.liquidate,
+    classifiers={
+        "liquidationCall(address,address,address,uint256,bool)": AaveLiquidationClassifier,
     },
 )
 
-AAVE_CLASSIFIER_SPECS = [AAVE_SPEC]
+ATOKENS_SPEC = ClassifierSpec(
+    abi_name="aTokens",
+    protocol=Protocol.aave,
+    classifiers={
+        "transferOnLiquidation(address,address,uint256)": AaveTransferClassifier,
+    },
+)
+
+AAVE_CLASSIFIER_SPECS: List[ClassifierSpec] = [AAVE_SPEC, ATOKENS_SPEC]

mev_inspect/classifiers/specs/balancer.py

@@ -0,0 +1,41 @@
+from typing import List, Optional
+
+from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
+from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+BALANCER_V1_POOL_ABI_NAME = "BPool"
+
+
+class BalancerSwapClassifier(SwapClassifier):
+    @staticmethod
+    def parse_swap(
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
+
+        recipient_address = trace.from_address
+
+        swap = create_swap_from_pool_transfers(
+            trace, recipient_address, prior_transfers, child_transfers
+        )
+        return swap
+
+
+BALANCER_V1_SPECS = [
+    ClassifierSpec(
+        abi_name=BALANCER_V1_POOL_ABI_NAME,
+        protocol=Protocol.balancer_v1,
+        classifiers={
+            "swapExactAmountIn(address,uint256,address,uint256,uint256)": BalancerSwapClassifier,
+            "swapExactAmountOut(address,uint256,address,uint256,uint256)": BalancerSwapClassifier,
+        },
+    ),
+]
+
+BALANCER_CLASSIFIER_SPECS = [
+    *BALANCER_V1_SPECS,
+]

mev_inspect/classifiers/specs/bancor.py

@@ -0,0 +1,41 @@
+from typing import List, Optional
+
+from mev_inspect.classifiers.helpers import create_swap_from_recipient_transfers
+from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+BANCOR_NETWORK_ABI_NAME = "BancorNetwork"
+BANCOR_NETWORK_CONTRACT_ADDRESS = "0x2F9EC37d6CcFFf1caB21733BdaDEdE11c823cCB0"
+
+
+class BancorSwapClassifier(SwapClassifier):
+    @staticmethod
+    def parse_swap(
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
+        recipient_address = trace.from_address
+
+        swap = create_swap_from_recipient_transfers(
+            trace,
+            BANCOR_NETWORK_CONTRACT_ADDRESS,
+            recipient_address,
+            prior_transfers,
+            child_transfers,
+        )
+        return swap
+
+
+BANCOR_NETWORK_SPEC = ClassifierSpec(
+    abi_name=BANCOR_NETWORK_ABI_NAME,
+    protocol=Protocol.bancor,
+    classifiers={
+        "convertByPath(address[],uint256,uint256,address,address,uint256)": BancorSwapClassifier,
+    },
+    valid_contract_addresses=[BANCOR_NETWORK_CONTRACT_ADDRESS],
+)
+
+BANCOR_CLASSIFIER_SPECS = [BANCOR_NETWORK_SPEC]

mev_inspect/classifiers/specs/compound.py

@@ -0,0 +1,127 @@
+from typing import List, Optional
+
+from mev_inspect.classifiers.helpers import get_debt_transfer, get_received_transfer
+from mev_inspect.schemas.classifiers import (
+    Classification,
+    ClassifiedTrace,
+    ClassifierSpec,
+    DecodedCallTrace,
+    LiquidationClassifier,
+    SeizeClassifier,
+)
+from mev_inspect.schemas.liquidations import Liquidation
+from mev_inspect.schemas.prices import CETH_TOKEN_ADDRESS, ETH_TOKEN_ADDRESS
+from mev_inspect.schemas.traces import Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+
+class CompoundLiquidationClassifier(LiquidationClassifier):
+    @staticmethod
+    def parse_liquidation(
+        liquidation_trace: DecodedCallTrace,
+        child_transfers: List[Transfer],
+        child_traces: List[ClassifiedTrace],
+    ) -> Optional[Liquidation]:
+
+        liquidator = liquidation_trace.from_address
+        liquidated = liquidation_trace.inputs["borrower"]
+
+        debt_token_address = liquidation_trace.to_address
+        received_token_address = liquidation_trace.inputs["cTokenCollateral"]
+
+        debt_purchase_amount = None
+        received_amount = None
+
+        debt_purchase_amount, debt_token_address = (
+            (liquidation_trace.value, ETH_TOKEN_ADDRESS)
+            if debt_token_address == CETH_TOKEN_ADDRESS and liquidation_trace.value != 0
+            else (liquidation_trace.inputs["repayAmount"], CETH_TOKEN_ADDRESS)
+        )
+
+        debt_transfer = get_debt_transfer(liquidator, child_transfers)
+
+        received_transfer = get_received_transfer(liquidator, child_transfers)
+
+        seize_trace = _get_seize_call(child_traces)
+
+        if debt_transfer is not None:
+            debt_token_address = debt_transfer.token_address
+            debt_purchase_amount = debt_transfer.amount
+
+        if received_transfer is not None:
+            received_token_address = received_transfer.token_address
+            received_amount = received_transfer.amount
+
+        elif seize_trace is not None and seize_trace.inputs is not None:
+            received_amount = seize_trace.inputs["seizeTokens"]
+
+        if received_amount is None:
+            return None
+
+        return Liquidation(
+            liquidated_user=liquidated,
+            debt_token_address=debt_token_address,
+            liquidator_user=liquidator,
+            debt_purchase_amount=debt_purchase_amount,
+            protocol=liquidation_trace.protocol,
+            received_amount=received_amount,
+            received_token_address=received_token_address,
+            transaction_hash=liquidation_trace.transaction_hash,
+            trace_address=liquidation_trace.trace_address,
+            block_number=liquidation_trace.block_number,
+            error=liquidation_trace.error,
+        )
+
+        return None
+
+
+COMPOUND_V2_CETH_SPEC = ClassifierSpec(
+    abi_name="CEther",
+    protocol=Protocol.compound_v2,
+    valid_contract_addresses=["0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5"],
+    classifiers={
+        "liquidateBorrow(address,address)": CompoundLiquidationClassifier,
+        "seize(address,address,uint256)": SeizeClassifier,
+    },
+)
+
+COMPOUND_V2_CTOKEN_SPEC = ClassifierSpec(
+    abi_name="CToken",
+    protocol=Protocol.compound_v2,
+    valid_contract_addresses=[
+        "0x6c8c6b02e7b2be14d4fa6022dfd6d75921d90e4e",
+        "0x5d3a536e4d6dbd6114cc1ead35777bab948e3643",
+        "0x158079ee67fce2f58472a96584a73c7ab9ac95c1",
+        "0x39aa39c021dfbae8fac545936693ac917d5e7563",
+        "0xf650c3d88d12db855b8bf7d11be6c55a4e07dcc9",
+        "0xc11b1268c1a384e55c48c2391d8d480264a3a7f4",
+        "0xb3319f5d18bc0d84dd1b4825dcde5d5f7266d407",
+        "0xf5dce57282a584d2746faf1593d3121fcac444dc",
+        "0x35a18000230da775cac24873d00ff85bccded550",
+        "0x70e36f6bf80a52b3b46b3af8e106cc0ed743e8e4",
+        "0xccf4429db6322d5c611ee964527d42e5d685dd6a",
+        "0x12392f67bdf24fae0af363c24ac620a2f67dad86",
+        "0xface851a4921ce59e912d19329929ce6da6eb0c7",
+        "0x95b4ef2869ebd94beb4eee400a99824bf5dc325b",
+        "0x4b0181102a0112a2ef11abee5563bb4a3176c9d7",
+        "0xe65cdb6479bac1e22340e4e755fae7e509ecd06c",
+        "0x80a2ae356fc9ef4305676f7a3e2ed04e12c33946",
+    ],
+    classifiers={
+        "liquidateBorrow(address,uint256,address)": CompoundLiquidationClassifier,
+        "seize(address,address,uint256)": SeizeClassifier,
+    },
+)
+
+COMPOUND_CLASSIFIER_SPECS: List[ClassifierSpec] = [
+    COMPOUND_V2_CETH_SPEC,
+    COMPOUND_V2_CTOKEN_SPEC,
+]
+
+
+def _get_seize_call(traces: List[ClassifiedTrace]) -> Optional[ClassifiedTrace]:
+    """Find the call to `seize` in the child traces (successful liquidation)"""
+    for trace in traces:
+        if trace.classification == Classification.seize:
+            return trace
+    return None

mev_inspect/classifiers/specs/cream.py

@@ -0,0 +1,204 @@
+from typing import List, Optional
+
+from mev_inspect.classifiers.helpers import get_debt_transfer, get_received_transfer
+from mev_inspect.schemas.classifiers import (
+    Classification,
+    ClassifiedTrace,
+    ClassifierSpec,
+    DecodedCallTrace,
+    LiquidationClassifier,
+    SeizeClassifier,
+)
+from mev_inspect.schemas.liquidations import Liquidation
+from mev_inspect.schemas.prices import ETH_TOKEN_ADDRESS
+from mev_inspect.schemas.traces import Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+CRETH_TOKEN_ADDRESS = "0xd06527d5e56a3495252a528c4987003b712860ee"
+
+
+class CreamLiquidationClassifier(LiquidationClassifier):
+    @staticmethod
+    def parse_liquidation(
+        liquidation_trace: DecodedCallTrace,
+        child_transfers: List[Transfer],
+        child_traces: List[ClassifiedTrace],
+    ) -> Optional[Liquidation]:
+
+        liquidator = liquidation_trace.from_address
+        liquidated = liquidation_trace.inputs["borrower"]
+
+        debt_token_address = liquidation_trace.to_address
+        received_token_address = liquidation_trace.inputs["cTokenCollateral"]
+
+        debt_purchase_amount = None
+        received_amount = None
+
+        debt_purchase_amount, debt_token_address = (
+            (liquidation_trace.value, ETH_TOKEN_ADDRESS)
+            if debt_token_address == CRETH_TOKEN_ADDRESS
+            and liquidation_trace.value != 0
+            else (liquidation_trace.inputs["repayAmount"], CRETH_TOKEN_ADDRESS)
+        )
+
+        debt_transfer = get_debt_transfer(liquidator, child_transfers)
+
+        received_transfer = get_received_transfer(liquidator, child_transfers)
+
+        seize_trace = _get_seize_call(child_traces)
+
+        if debt_transfer is not None:
+            debt_token_address = debt_transfer.token_address
+            debt_purchase_amount = debt_transfer.amount
+
+        if received_transfer is not None:
+            received_token_address = received_transfer.token_address
+            received_amount = received_transfer.amount
+
+        elif seize_trace is not None and seize_trace.inputs is not None:
+            received_amount = seize_trace.inputs["seizeTokens"]
+
+        if received_amount is None:
+            return None
+
+        return Liquidation(
+            liquidated_user=liquidated,
+            debt_token_address=debt_token_address,
+            liquidator_user=liquidator,
+            debt_purchase_amount=debt_purchase_amount,
+            protocol=liquidation_trace.protocol,
+            received_amount=received_amount,
+            received_token_address=received_token_address,
+            transaction_hash=liquidation_trace.transaction_hash,
+            trace_address=liquidation_trace.trace_address,
+            block_number=liquidation_trace.block_number,
+            error=liquidation_trace.error,
+        )
+
+        return None
+
+
+CREAM_CRETH_SPEC = ClassifierSpec(
+    abi_name="CEther",
+    protocol=Protocol.cream,
+    valid_contract_addresses=["0xD06527D5e56A3495252A528C4987003b712860eE"],
+    classifiers={
+        "liquidateBorrow(address,address)": CreamLiquidationClassifier,
+        "seize(address,address,uint256)": SeizeClassifier,
+    },
+)
+
+CREAM_CTOKEN_SPEC = ClassifierSpec(
+    abi_name="CToken",
+    protocol=Protocol.cream,
+    valid_contract_addresses=[
+        "0xd06527d5e56a3495252a528c4987003b712860ee",
+        "0x51f48b638f82e8765f7a26373a2cb4ccb10c07af",
+        "0x44fbebd2f576670a6c33f6fc0b00aa8c5753b322",
+        "0xcbae0a83f4f9926997c8339545fb8ee32edc6b76",
+        "0xce4fe9b4b8ff61949dcfeb7e03bc9faca59d2eb3",
+        "0x19d1666f543d42ef17f66e376944a22aea1a8e46",
+        "0x9baf8a5236d44ac410c0186fe39178d5aad0bb87",
+        "0x797aab1ce7c01eb727ab980762ba88e7133d2157",
+        "0x892b14321a4fcba80669ae30bd0cd99a7ecf6ac0",
+        "0x697256caa3ccafd62bb6d3aa1c7c5671786a5fd9",
+        "0x8b86e0598616a8d4f1fdae8b59e55fb5bc33d0d6",
+        "0xc7fd8dcee4697ceef5a2fd4608a7bd6a94c77480",
+        "0x17107f40d70f4470d20cb3f138a052cae8ebd4be",
+        "0x1ff8cdb51219a8838b52e9cac09b71e591bc998e",
+        "0x3623387773010d9214b10c551d6e7fc375d31f58",
+        "0x4ee15f44c6f0d8d1136c83efd2e8e4ac768954c6",
+        "0x338286c0bc081891a4bda39c7667ae150bf5d206",
+        "0x10fdbd1e48ee2fd9336a482d746138ae19e649db",
+        "0x01da76dea59703578040012357b81ffe62015c2d",
+        "0xef58b2d5a1b8d3cde67b8ab054dc5c831e9bc025",
+        "0xe89a6d0509faf730bd707bf868d9a2a744a363c7",
+        "0xeff039c3c1d668f408d09dd7b63008622a77532c",
+        "0x22b243b96495c547598d9042b6f94b01c22b2e9e",
+        "0x8b3ff1ed4f36c2c2be675afb13cc3aa5d73685a5",
+        "0x2a537fa9ffaea8c1a41d3c2b68a9cb791529366d",
+        "0x7ea9c63e216d5565c3940a2b3d150e59c2907db3",
+        "0x3225e3c669b39c7c8b3e204a8614bb218c5e31bc",
+        "0xf55bbe0255f7f4e70f63837ff72a577fbddbe924",
+        "0x903560b1cce601794c584f58898da8a8b789fc5d",
+        "0x054b7ed3f45714d3091e82aad64a1588dc4096ed",
+        "0xd5103afcd0b3fa865997ef2984c66742c51b2a8b",
+        "0xfd609a03b393f1a1cfcacedabf068cad09a924e2",
+        "0xd692ac3245bb82319a31068d6b8412796ee85d2c",
+        "0x92b767185fb3b04f881e3ac8e5b0662a027a1d9f",
+        "0x10a3da2bb0fae4d591476fd97d6636fd172923a8",
+        "0x3c6c553a95910f9fc81c98784736bd628636d296",
+        "0x21011bc93d9e515b9511a817a1ed1d6d468f49fc",
+        "0x85759961b116f1d36fd697855c57a6ae40793d9b",
+        "0x7c3297cfb4c4bbd5f44b450c0872e0ada5203112",
+        "0x7aaa323d7e398be4128c7042d197a2545f0f1fea",
+        "0x011a014d5e8eb4771e575bb1000318d509230afa",
+        "0xe6c3120f38f56deb38b69b65cc7dcaf916373963",
+        "0x4fe11bc316b6d7a345493127fbe298b95adaad85",
+        "0xcd22c4110c12ac41acefa0091c432ef44efaafa0",
+        "0x228619cca194fbe3ebeb2f835ec1ea5080dafbb2",
+        "0x73f6cba38922960b7092175c0add22ab8d0e81fc",
+        "0x38f27c03d6609a86ff7716ad03038881320be4ad",
+        "0x5ecad8a75216cea7dff978525b2d523a251eea92",
+        "0x5c291bc83d15f71fb37805878161718ea4b6aee9",
+        "0x6ba0c66c48641e220cf78177c144323b3838d375",
+        "0xd532944df6dfd5dd629e8772f03d4fc861873abf",
+        "0x197070723ce0d3810a0e47f06e935c30a480d4fc",
+        "0xc25eae724f189ba9030b2556a1533e7c8a732e14",
+        "0x25555933a8246ab67cbf907ce3d1949884e82b55",
+        "0xc68251421edda00a10815e273fa4b1191fac651b",
+        "0x65883978ada0e707c3b2be2a6825b1c4bdf76a90",
+        "0x8b950f43fcac4931d408f1fcda55c6cb6cbf3096",
+        "0x59089279987dd76fc65bf94cb40e186b96e03cb3",
+        "0x2db6c82ce72c8d7d770ba1b5f5ed0b6e075066d6",
+        "0xb092b4601850e23903a42eacbc9d8a0eec26a4d5",
+        "0x081fe64df6dc6fc70043aedf3713a3ce6f190a21",
+        "0x1d0986fb43985c88ffa9ad959cc24e6a087c7e35",
+        "0xc36080892c64821fa8e396bc1bd8678fa3b82b17",
+        "0x8379baa817c5c5ab929b03ee8e3c48e45018ae41",
+        "0x299e254a8a165bbeb76d9d69305013329eea3a3b",
+        "0xf8445c529d363ce114148662387eba5e62016e20",
+        "0x28526bb33d7230e65e735db64296413731c5402e",
+        "0x45406ba53bb84cd32a58e7098a2d4d1b11b107f6",
+        "0x6d1b9e01af17dd08d6dec08e210dfd5984ff1c20",
+        "0x1f9b4756b008106c806c7e64322d7ed3b72cb284",
+        "0xab10586c918612ba440482db77549d26b7abf8f7",
+        "0xdfff11dfe6436e42a17b86e7f419ac8292990393",
+        "0xdbb5e3081def4b6cdd8864ac2aeda4cbf778fecf",
+        "0x71cefcd324b732d4e058afacba040d908c441847",
+        "0x1a122348b73b58ea39f822a89e6ec67950c2bbd0",
+        "0x523effc8bfefc2948211a05a905f761cba5e8e9e",
+        "0x4202d97e00b9189936edf37f8d01cff88bdd81d4",
+        "0x4baa77013ccd6705ab0522853cb0e9d453579dd4",
+        "0x98e329eb5aae2125af273102f3440de19094b77c",
+        "0x8c3b7a4320ba70f8239f83770c4015b5bc4e6f91",
+        "0xe585c76573d7593abf21537b607091f76c996e73",
+        "0x81e346729723c4d15d0fb1c5679b9f2926ff13c6",
+        "0x766175eac1a99c969ddd1ebdbe7e270d508d8fff",
+        "0xd7394428536f63d5659cc869ef69d10f9e66314b",
+        "0x1241b10e7ea55b22f5b2d007e8fecdf73dcff999",
+        "0x2a867fd776b83e1bd4e13c6611afd2f6af07ea6d",
+        "0x250fb308199fe8c5220509c1bf83d21d60b7f74a",
+        "0x4112a717edd051f77d834a6703a1ef5e3d73387f",
+        "0xf04ce2e71d32d789a259428ddcd02d3c9f97fb4e",
+        "0x89e42987c39f72e2ead95a8a5bc92114323d5828",
+        "0x58da9c9fc3eb30abbcbbab5ddabb1e6e2ef3d2ef",
+    ],
+    classifiers={
+        "liquidateBorrow(address,uint256,address)": CreamLiquidationClassifier,
+        "seize(address,address,uint256)": SeizeClassifier,
+    },
+)
+
+CREAM_CLASSIFIER_SPECS: List[ClassifierSpec] = [
+    CREAM_CRETH_SPEC,
+    CREAM_CTOKEN_SPEC,
+]
+
+
+def _get_seize_call(traces: List[ClassifiedTrace]) -> Optional[ClassifiedTrace]:
+    """Find the call to `seize` in the child traces (successful liquidation)"""
+    for trace in traces:
+        if trace.classification == Classification.seize:
+            return trace
+    return None

mev_inspect/classifiers/specs/cryptopunks.py

@@ -0,0 +1,27 @@
+from mev_inspect.schemas.classifiers import Classifier, ClassifierSpec
+from mev_inspect.schemas.traces import Classification, Protocol
+
+
+class PunkBidAcceptanceClassifier(Classifier):
+    @staticmethod
+    def get_classification() -> Classification:
+        return Classification.punk_accept_bid
+
+
+class PunkBidClassifier(Classifier):
+    @staticmethod
+    def get_classification() -> Classification:
+        return Classification.punk_bid
+
+
+CRYPTO_PUNKS_SPEC = ClassifierSpec(
+    abi_name="cryptopunks",
+    protocol=Protocol.cryptopunks,
+    valid_contract_addresses=["0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB"],
+    classifiers={
+        "enterBidForPunk(uint256)": PunkBidClassifier,
+        "acceptBidForPunk(uint256,uint256)": PunkBidAcceptanceClassifier,
+    },
+)
+
+CRYPTOPUNKS_CLASSIFIER_SPECS = [CRYPTO_PUNKS_SPEC]

mev_inspect/classifiers/specs/curve.py

@@ -1,29 +1,28 @@
-from mev_inspect.schemas.classified_traces import (
-    ClassifierSpec,
-    Protocol,
-)
+from typing import List, Optional
+
+from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
+from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+
+class CurveSwapClassifier(SwapClassifier):
+    @staticmethod
+    def parse_swap(
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
+
+        recipient_address = trace.from_address
+
+        swap = create_swap_from_pool_transfers(
+            trace, recipient_address, prior_transfers, child_transfers
+        )
+        return swap
 
-"""
-Deployment addresses found here
-https://curve.readthedocs.io/ref-addresses.html
 
-organized into 3 groups
-1. Base Pools: 2 or more tokens implementing stable swap
-  - StableSwap<pool>
-  - Deposit<pool>
-  - CurveContract<version>
-  - CurveTokenV1/V2
-2. Meta Pools: 1 token trading with an LP from above
-  - StableSwap<pool>
-  - Deposit<pool>
-  - CurveTokenV1/V2
-3. Liquidity Gauges: stake LP get curve governance token?
-  - LiquidityGauge
-  - LiquidityGaugeV1/V2
-  - LiquidityGaugeReward
-4. DAO stuff
-5..? Other stuff, haven't decided if important
-"""
 CURVE_BASE_POOLS = [
     ClassifierSpec(
         abi_name="CurveTokenV1",
@@ -72,101 +71,171 @@ CURVE_BASE_POOLS = [
         abi_name="StableSwap3Pool",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xbEbc44782C7dB0a1A60Cb6fe97d0b483032FF1C7"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapAAVE",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xDeBF20617708857ebe4F679508E7b7863a8A8EeE"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapAETH",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xA96A65c051bF88B4095Ee1f2451C2A9d43F53Ae2"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapBUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x79a8C46DeA5aDa233ABaFFD40F3A0A2B1e5A4F27"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapCompound",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xA2B47E3D5c44877cca798226B7B8118F9BFb7A56"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapEURS",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x0Ce6a5fF5217e38315f87032CF90686C96627CAA"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwaphBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x4CA9b3063Ec5866A4B82E437059D2C43d1be596F"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapIronBank",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x2dded6Da1BF5DBdF597C45fcFaa3194e53EcfeAF"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapLink",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xf178c0b5bb7e7abf4e12a4838c7b7c5ba2c623c0"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapPAX",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x06364f10B501e868329afBc005b3492902d6C763"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwaprenBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x93054188d876f558f4a66B2EF1d97d16eDf0895B"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwaprETH",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xF9440930043eb3997fc70e1339dBb11F341de7A8"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapsAAVE",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xEB16Ae0052ed37f479f7fe63849198Df1765a733"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapsBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x7fC77b5c7614E1533320Ea6DDc2Eb61fa00A9714"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapsETH",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xc5424B857f758E906013F3555Dad202e4bdB4567"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapstETH",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xDC24316b9AE028F1497c275EB9192a3Ea0f67022"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapsUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xA5407eAE9Ba41422680e2e00537571bcC53efBfD"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapUSDT",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x52EA46506B9CC5Ef470C5bf89f17Dc28bB35D85C"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapY",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x45F783CCE6B7FF23B2ab2D70e416cdb7D6055f51"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapYv2",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x8925D9d9B4569D737a48499DeF3f67BaA5a144b9"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="DepositBUSD",
@@ -300,51 +369,91 @@ CURVE_META_POOLS = [
         abi_name="StableSwapbBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x071c661B4DeefB59E2a3DdB20Db036821eeE8F4b"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapDUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x8038C01A0390a8c547446a0b2c18fc9aEFEcc10c"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapGUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x4f062658EaAF2C1ccf8C8e36D6824CDf41167956"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapHUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x3eF6A01A0f81D6046290f3e2A8c5b843e738E604"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapLinkUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xE7a24EF0C5e95Ffb0f6684b813A78F2a3AD7D171"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapMUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x8474DdbE98F5aA3179B3B3F5942D724aFcdec9f6"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapoBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xd81dA8D904b52208541Bade1bD6595D8a251F8dd"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwappBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x7F55DDe206dbAD629C080068923b36fe9D6bDBeF"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapRSV",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xC18cC39da8b11dA8c3541C598eE022258F9744da"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwaptBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xC25099792E9349C7DD09759744ea681C7de2cb66"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapUSD",
@@ -353,82 +462,29 @@ CURVE_META_POOLS = [
             "0x3E01dD8a5E1fb3481F0F589056b428Fc308AF0Fb",
             "0x0f9cb53Ebe405d49A0bbdBD291A65Ff571bC83e1",
         ],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapUSDP",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x42d7025938bEc20B69cBae5A77421082407f053A"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapUST",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x890f4e345B1dAED0367A877a1612f86A1f86985f"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
 ]
 
-"""
-CURVE_LIQUIDITY_GAUGES = [
-    ClassifierSpec(
-        abi_name="LiquidityGauge",
-        protocol=Protocol.curve,
-        valid_contract_addresses=[
-            "0xbFcF63294aD7105dEa65aA58F8AE5BE2D9d0952A",  # 3Pool
-            "0x69Fb7c45726cfE2baDeE8317005d3F94bE838840",  # BUSD
-            "0x7ca5b0a2910B33e9759DC7dDB0413949071D7575",  # Compound
-            "0xC5cfaDA84E902aD92DD40194f0883ad49639b023",  # GUSD
-            "0x4c18E409Dc8619bFb6a1cB56D114C3f592E0aE79",  # hBTC
-            "0x2db0E83599a91b508Ac268a6197b8B14F5e72840",  # HUSD
-            "0x64E3C23bfc40722d3B649844055F1D51c1ac041d",  # PAX
-            "0xB1F2cdeC61db658F091671F5f199635aEF202CAC",  # renBTC
-            "0xC2b1DF84112619D190193E48148000e3990Bf627",  # USDK
-            "0xF98450B5602fa59CC66e1379DFfB6FDDc724CfC4",  # USDN
-            "0xBC89cd85491d81C6AD2954E6d0362Ee29fCa8F53",  # USDT
-            "0xFA712EE4788C042e2B7BB55E6cb8ec569C4530c1",  # Y
-        ],
-    ),
-    ClassifierSpec(
-        abi_name="LiquidityGaugeV2",
-        protocol=Protocol.curve,
-        valid_contract_addresses=[
-            "0xd662908ADA2Ea1916B3318327A97eB18aD588b5d",  # AAVE
-            "0x6d10ed2cF043E6fcf51A0e7b4C2Af3Fa06695707",  # ankrETH
-            "0xdFc7AdFa664b08767b735dE28f9E84cd30492aeE",  # bBTC
-            "0x90Bb609649E0451E5aD952683D64BD2d1f245840",  # EURS
-            "0x72e158d38dbd50a483501c24f792bdaaa3e7d55c",  # FRAX
-            "0x11137B10C210b579405c21A07489e28F3c040AB1",  # oBTC
-            "0xF5194c3325202F456c95c1Cf0cA36f8475C1949F",  # IronBank
-            "0xFD4D8a17df4C27c1dD245d153ccf4499e806C87D",  # Link
-            "0xd7d147c6Bb90A718c3De8C0568F9B560C79fa416",  # pBTC
-            "0x462253b8F74B72304c145DB0e4Eebd326B22ca39",  # sAAVE
-            "0x3C0FFFF15EA30C35d7A85B85c0782D6c94e1d238",  # sETH
-            "0x182B723a58739a9c974cFDB385ceaDb237453c28",  # stETH
-            "0x055be5DDB7A925BfEF3417FC157f53CA77cA7222",  # USDP
-            "0x3B7020743Bc2A4ca9EaF9D0722d42E20d6935855",  # UST
-            "0x8101E6760130be2C8Ace79643AB73500571b7162",  # Yv2
-        ],
-    ),
-    ClassifierSpec(
-        abi_name="LiquidityGaugeV3",
-        protocol=Protocol.curve,
-        valid_contract_addresses=[
-            "0x9582C4ADACB3BCE56Fea3e590F05c3ca2fb9C477",  # alUSD
-            "0x824F13f1a2F29cFEEa81154b46C0fc820677A637",  # rETH
-            "0x6955a55416a06839309018A8B0cB72c4DDC11f15",  # TriCrypto
-        ],
-    ),
-    ClassifierSpec(
-        abi_name="LiquidityGaugeReward",
-        protocol=Protocol.curve,
-        valid_contract_addresses=[
-            "0xAEA6c312f4b3E04D752946d329693F7293bC2e6D",  # DUSD
-            "0x5f626c30EC1215f4EdCc9982265E8b1F411D1352",  # MUSD
-            "0x4dC4A289a8E33600D8bD4cf5F6313E43a37adec7",  # RSV
-            "0x705350c4BcD35c9441419DdD5d2f097d7a55410F",  # sBTC
-            "0xA90996896660DEcC6E997655E065b23788857849",  # sUSDv2
-            "0x6828bcF74279eE32f2723eC536c22c51Eed383C6",  # tBTC
-        ],
-    ),
-]
-"""
-
 CURVE_CLASSIFIER_SPECS = [*CURVE_BASE_POOLS, *CURVE_META_POOLS]

mev_inspect/classifiers/specs/erc20.py

@@ -1,15 +1,27 @@
-from mev_inspect.schemas.classified_traces import (
-    Classification,
-    ClassifierSpec,
-)
+from mev_inspect.schemas.classifiers import ClassifierSpec, TransferClassifier
+from mev_inspect.schemas.traces import DecodedCallTrace
+from mev_inspect.schemas.transfers import Transfer
+
+
+class ERC20TransferClassifier(TransferClassifier):
+    @staticmethod
+    def get_transfer(trace: DecodedCallTrace) -> Transfer:
+        return Transfer(
+            block_number=trace.block_number,
+            transaction_hash=trace.transaction_hash,
+            trace_address=trace.trace_address,
+            amount=trace.inputs["amount"],
+            to_address=trace.inputs["recipient"],
+            from_address=trace.inputs.get("sender", trace.from_address),
+            token_address=trace.to_address,
+        )
 
 
 ERC20_SPEC = ClassifierSpec(
     abi_name="ERC20",
-    classifications={
-        "transferFrom(address,address,uint256)": Classification.transfer,
-        "transfer(address,uint256)": Classification.transfer,
-        "burn(address)": Classification.burn,
+    classifiers={
+        "transferFrom(address,address,uint256)": ERC20TransferClassifier,
+        "transfer(address,uint256)": ERC20TransferClassifier,
     },
 )
 

mev_inspect/classifiers/specs/opensea.py

@@ -0,0 +1,42 @@
+from typing import List, Optional
+
+from mev_inspect.classifiers.helpers import create_nft_trade_from_transfers
+from mev_inspect.schemas.classifiers import ClassifierSpec, NftTradeClassifier
+from mev_inspect.schemas.nft_trades import NftTrade
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+OPENSEA_WALLET_ADDRESS = "0x5b3256965e7c3cf26e11fcaf296dfc8807c01073"
+
+
+class OpenseaClassifier(NftTradeClassifier):
+    @staticmethod
+    def parse_trade(
+        trace: DecodedCallTrace,
+        child_transfers: List[Transfer],
+    ) -> Optional[NftTrade]:
+        addresses = trace.inputs["addrs"]
+        buy_maker = addresses[1]
+        sell_maker = addresses[8]
+        target = addresses[4]
+
+        return create_nft_trade_from_transfers(
+            trace,
+            child_transfers,
+            collection_address=target,
+            seller_address=sell_maker,
+            buyer_address=buy_maker,
+            exchange_wallet_address=OPENSEA_WALLET_ADDRESS,
+        )
+
+
+OPENSEA_SPEC = ClassifierSpec(
+    abi_name="WyvernExchange",
+    protocol=Protocol.opensea,
+    valid_contract_addresses=["0x7be8076f4ea4a4ad08075c2508e481d6c946d12b"],
+    classifiers={
+        "atomicMatch_(address[14],uint256[18],uint8[8],bytes,bytes,bytes,bytes,bytes,bytes,uint8[2],bytes32[5])": OpenseaClassifier,
+    },
+)
+
+OPENSEA_CLASSIFIER_SPECS = [OPENSEA_SPEC]