No pass

.dockerignore

@@ -1 +1,2 @@
+tests
 cache

.env

@@ -0,0 +1,8 @@
+# Postgres
+POSTGRES_SERVER=db
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=password
+POSTGRES_DB=mev_inspect
+
+# SQLAlchemy
+SQLALCHEMY_DATABASE_URI=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_SERVER/$POSTGRES_DB

.github/workflows/github-actions.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Bootstrap poetry
         shell: bash
         run: |
-          curl -sSL https://install.python-poetry.org \
+          curl -sL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py \
             | python - -y 
 
       - name: Update PATH
@@ -51,8 +51,8 @@ jobs:
 
       - name: Run precommit
         run: |
-          poetry run pre-commit run --all-files
+          poetry run pre-commit
 
       - name: Test with pytest
         shell: bash
-        run: poetry run pytest --cov=mev_inspect tests
+        run: poetry run test

.gitignore

@@ -19,12 +19,3 @@ cache
 
 # k8s
 .helm
-
-# env
-.envrc
-
-# pycharm
-.idea
-
-.env
-.python-version

.pre-commit-config.yaml

@@ -1,16 +1,9 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: 22.3.0
+    rev: 20.8b1
     hooks:
-      - id: black
-        language_version: python3.9
--   repo: local
-    hooks:
-      - id: isort
-        name: isort
-        entry: poetry run isort .
-        language: system
-        types: [python]
+    - id: black
+      language_version: python3.9
 -   repo: local
     hooks:
     - id: pylint
@@ -20,7 +13,7 @@ repos:
       language: system
       types: [python]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.942
+    rev: v0.910
     hooks:
       - id: 'mypy'
         additional_dependencies:

.pylintrc

@@ -433,7 +433,7 @@ int-import-graph=
 known-standard-library=
 
 # Force import order to recognize a module as part of a third party library.
-known-third-party=alembic
+known-third-party=enchant
 
 # Couples of modules and preferred modules, separated by a comma.
 preferred-modules=

CONTRIBUTING.md

@@ -1,36 +0,0 @@
-# Contributing guide
-
-Welcome to the Flashbots collective! We just ask you to be nice when you play with us.
-
-## Pre-commit
-
-We use pre-commit to maintain a consistent style, prevent errors, and ensure test coverage.
-
-To set up, install dependencies through `poetry`:
-
-```
-poetry install
-```
-
-Then install pre-commit hooks with:
-
-```
-poetry run pre-commit install
-```
-
-## Tests
-
-Run tests with:
-
-```
-./mev test
-```
-
-## Send a pull request
-
-- Your proposed changes should be first described and discussed in an issue.
-- Open the branch in a personal fork, not in the team repository.
-- Every pull request should be small and represent a single change. If the problem is complicated, split it in multiple issues and pull requests.
-- Every pull request should be covered by unit tests.
-
-We appreciate you, friend <3.

Dockerfile

@@ -1,29 +1,21 @@
-FROM python:3.9-slim-buster
+FROM python:3.9
 
-ENV POETRY_VERSION=1.1.12
-
-RUN useradd --create-home flashbot \
+RUN pip install -U pip \
     && apt-get update \
-    && apt-get install -y --no-install-recommends build-essential libffi-dev libpq-dev gcc procps \
-    && pip install poetry==$POETRY_VERSION \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
+    && curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
 
-ENV PATH="${PATH}:/home/flashbot/.local/bin"
+ENV PATH="${PATH}:/root/.poetry/bin"
 
-COPY --chown=flashbot ./pyproject.toml /app/pyproject.toml
-COPY --chown=flashbot ./poetry.lock /app/poetry.lock
+COPY ./pyproject.toml /app/pyproject.toml
+COPY ./poetry.lock /app/poetry.lock
 WORKDIR /app/
 
-USER flashbot
+RUN poetry config virtualenvs.create false && \
+    poetry install
 
-RUN poetry config virtualenvs.create false \
-    && poetry install
-
-COPY --chown=flashbot . /app
+COPY . /app
 
 # easter eggs 😝
 RUN echo "PS1='🕵️:\[\033[1;36m\]\h \[\033[1;34m\]\W\[\033[0;35m\]\[\033[1;36m\]$ \[\033[0m\]'" >> ~/.bashrc
 
-ENTRYPOINT [ "poetry" ]
-CMD [ "run", "python", "loop.py" ]
+ENTRYPOINT [ "/app/run.sh"]

LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2023 Flashbots
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

MONOLITHIC.md

@@ -1,126 +0,0 @@
-
-# Running mev-inspect-py without kubernetes ('monolithic mode')
-
-Running mev-inspect-py outside of kubernetes can be useful for debug purposes. In this case, the steps for installation are:
-1. Install dependencies (pyenv, poetry, postgres)
-1. Set up python virtual environment using matching python version (3.9.x) and install required python modules using poetry
-1. Create postgres database
-1. Run database migrations
-
-The database credentials and archive node address used by mev-inspect-py need to be loaded into environment variables (both for database migrations and to run mev-inspect-py).
-
-## Ubuntu install instructions
-
-So, starting from a clean Ubuntu 22.04 installation, the prerequisites for pyenv, psycopg2 (python3-dev libpq-dev) can be installed with
-
-`sudo apt install -y make build-essential git libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev liblzma-dev python3-dev libpq-dev`
-
-### pyenv
-Install pyenv using the web installer
-
-`curl https://pyenv.run | bash`
-
-and add the following to `~/.bashrc` (if running locally) or `~/.profile` (if running over ssh).
-
-```
-export PYENV_ROOT="$HOME/.pyenv"
-command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"
-eval "$(pyenv init -)"
-```
-
-Then update the current shell by running `source ~/.bashrc` or `source ~/.profile` as appropriate.
-
-### Poetry
-
-Install Poetry using the web installer
-
-`curl -sSL https://install.python-poetry.org | python3 -`
-
-add the following to `~/.bashrc` (if running locally) or `~/.profile` (if running over ssh)
-
-`export PATH="/home/user/.local/bin:$PATH"`
-
-If running over ssh you should also add the following to `~/.profile` to prevent [Poetry errors](https://github.com/python-poetry/poetry/issues/1917) from a lack of active keyring:
-
-`export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring`
-
-Again update current shell by running `source ~/.bashrc` or `source ~/.profile` as appropriate.
-
-### postgres
-We have tested two alternatives for postgres - installing locally or as a container.
-
-#### Option 1: Installing locally
-
-To install locally from a clean Ubuntu 22.04 installation, run:
-`sudo apt install postgresql postgresql-contrib`
-
-Note: You may need to reconfigure your pg-hba.conf to allow local access.
-
-#### Option 2: Installing docker
-
-To avoid interfering with your local postgres instance, you may prefer to run postgres within a docker container.
-For docker installation instructions, please refer to https://docs.docker.com/engine/install/ubuntu/
-
-### mev-inspect-py
-
-With all dependencies now installed, clone the mev-inspec-py repo
-```
-git clone https://github.com/flashbots/mev-inspect-py.git
-cd mev-inspect-py
-```
-We now install the required pythn version and use Poetry to install the required python modules into a virtual environment. 
-
-```
-pyenv install 3.9.16
-pyenv local 3.9.16
-poetry env use 3.9.16
-poetry install
-```
-
-### Create database
-mev-inspect-py outputs to a postgres database, so we need to set this up. There are various ways of doing this, two options are presented here.
-
-#### Option 1 — Run postgres locally
-```
-sudo -u postgres psql
-\password
-postgres
-create database mev_inspect;
-\q
-```
-
-#### Option 2 — Use postgres docker image
-To avoid interfering with your local postgres instance, you may prefer to run postgres within a docker container. First ensure that postgres is not currently running to ensure port `5432` is available:
-`sudo systemctl stop postgresql`
-and then start a containerised postgres instance:
-`sudo docker run -d -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=mev_inspect postgres`
-
-### Environment variables
-We will need to set a few environment variables to use mev-inspect-py. **These will be required every time mev-inspect-py runs**, so again you may wish to add these to your `~/.bashrc` and/or `~/.profile` as appropriate. Note that you need to substitute the correct URL for your archive node below if you are not running Erigon locally.
-```
-export POSTGRES_USER=postgres
-export POSTGRES_PASSWORD=postgres
-export POSTGRES_HOST=localhost
-export RPC_URL="http://127.0.0.1:8545"
-```
-### Database migrations
-Finally run the database migrations and fetch price information:
-
-```
-poetry run alembic upgrade head
-poetry run fetch-all-prices
-```
-
-## Usage instructions
-The same functionality available through kubernetes can be run in 'monolithic mode', but the relevant functions now need to be invoked by Poetry directly. So to inspect a single block, run for example:
-
-`poetry run inspect-block 16379706`
-
-Or to inspect a range of blocks:
-
-`poetry run inspect-many-blocks 16379606 16379706`
-
-Or to run the test suite:
-
-`poetry run pytest tests`
-

README.md

@@ -1,306 +1,104 @@
-⚠️ This tool has been deprecated. You can visit [Flashbots Data](https://datasets.flashbots.net/) for historical mev-inspect data on Ethereum and join us on the [Flashbots forum](https://collective.flashbots.net). ⚠️
+# mev-inspect
+A [WIP] Ethereum MEV Inspector in Python managed by Poetry
 
-# mev-inspect-py
+## Containers
+mev-inspect's local setup is built on [Docker Compose](https://docs.docker.com/compose/)
 
-[![standard-readme compliant](https://img.shields.io/badge/readme%20style-standard-brightgreen.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme)
-[![Discord](https://img.shields.io/discord/755466764501909692)](https://discord.gg/7hvTycdNcK)
+By default it starts up:
+- `mev-inspect` - a container with the code in this repo used for running scripts
+- `db` - a postgres database instance
+- `pgadmin` - a postgres DB UI for querying and more (avaiable at localhost:5050)
 
-[Maximal extractable value](https://ethereum.org/en/developers/docs/mev/) inspector for Ethereum, to illuminate the [dark forest](https://www.paradigm.xyz/2020/08/ethereum-is-a-dark-forest/) 🌲💡
-
-Given a block, mev-inspect finds:
-- miner payments (gas + coinbase)
-- tokens transfers and profit
-- swaps and [arbitrages](https://twitter.com/bertcmiller/status/1427632028263059462)
-- ...and more
-
-Data is stored in Postgres for analysis.
-
-## Install
-
-mev-inspect-py is built to run on kubernetes locally and in production.
-
-### Dependencies
-
-- [docker](https://www.docker.com/products/docker-desktop)
-- [kind](https://kind.sigs.k8s.io/docs/user/quick-start), or a similar tool for running local Kubernetes clusters
-- [kubectl](https://kubernetes.io/docs/tasks/tools/)
-- [helm](https://helm.sh/docs/intro/install/)
-- [tilt](https://docs.tilt.dev/install.html)
-
-### Set up
-
-Create a new cluster with:
+## Running locally
+Setup [Docker](https://www.docker.com/products/docker-desktop)
+Setup [Poetry](https://python-poetry.org/docs/#osx--linux--bashonwindows-install-instructions)
 
+Install dependencies through poetry
 ```
-kind create cluster
+poetry install
 ```
 
-Set an environment variable `RPC_URL` to an RPC for fetching blocks.
-
-mev-inspect-py currently requires a node with support for Erigon traces and receipts (not geth yet 😔).
-
-[pokt.network](https://www.pokt.network/)'s "Ethereum Mainnet Archival with trace calls" is a good hosted option.
-
-Example:
-
+Start the services (optionally as daemon)
 ```
-export RPC_URL="http://111.111.111.111:8546"
+poetry run start [-d]
+```
+
+Apply the latest migrations against the local DB:
+```
+poetry run exec alembic upgrade head
+``` 
+
+Run inspect on a block
+```
+poetry run inspect -b/--block-number 11931270 -r/--rpc 'http://111.11.11.111:8545/'
+``` 
+
+To stop the services (if running in the background, otherwise just ctrl+c)
+```
+poetry run stop
+```
+
+MEV container can be attached via
+```
+poetry run attach
+```
+
+Running additional compose commands are possible through standard `docker
+compose ...` calls.  Check `docker compose help` for more tools available
+
+## Executing scripts
+Any script can be run from the mev-inspect container like
+```
+poetry run exec <your command here>
+```
+
+For example
+```
+poetry run exec python examples/uniswap_inspect.py -block_number=123 -rpc='111.111.111'
+```
+
+### Poetry Scripts
+```bash
+# code check
+poetry run lint # linting via Pylint
+poetry run test # testing and code coverage with Pytest
+poetry run isort # fixing imports 
+poetry run mypy # type checking 
+poetry run black # style guide 
+poetry run pre-commit run --all-files # runs Black, PyLint and MyPy
+# docker management
+poetry run start [-d] # starts all services, optionally as a daemon
+poetry run stop # shutsdown all services or just ctrl + c if foreground
+poetry run build # rebuilds containers
+poetry run attach # enters the mev-inspect container in interactive mode
+# launches inspection script
+poetry run inspect -b/--block-number 11931270 -r/--rpc 'http://111.11.11.111:8545/'
 ```
 
 
-Next, start all services with:
-
+## Rebuilding containers
+After changes to the app's Dockerfile, rebuild with
 ```
-tilt up
+poetry run build
 ```
 
-Press "space" to see a browser of the services starting up.
+## Using PGAdmin
 
-On first startup, you'll need to apply database migrations with:
+1. Go to [localhost:5050](localhost:5050)
 
-```
-./mev exec alembic upgrade head
-```
+2. Login with the PGAdmin username and password in `.env`
 
-And load prices data
-
-```
-./mev prices fetch-all
-```
-
-## Monolithic (non-kubernetes) install instructions
-
-For an alternative means of running mev-inspect-py for smaller set-ups or debug purposes see the [monolithic install instructions](MONOLITHIC.md).
-
-## Usage
-
-### Inspect a single block
-
-Inspecting block [12914944](https://twitter.com/mevalphaleak/status/1420416437575901185):
-
-```
-./mev inspect 12914944
-```
-
-### Inspect many blocks
-
-Inspecting blocks 12914944 to 12914954:
-
-```
-./mev inspect-many 12914944 12914954
-```
-
-### Inspect all incoming blocks
-
-Start a block listener with:
-
-```
-./mev listener start
-```
-
-By default, it will pick up wherever you left off.
-If running for the first time, listener starts at the latest block.
-
-Tail logs for the listener with:
-
-```
-./mev listener tail
-```
-
-And stop the listener with:
-
-```
-./mev listener stop
-```
-
-### Backfilling
-
-For larger backfills, you can inspect many blocks in parallel
-
-To inspect blocks 12914944 to 12915044, run
-```
-./mev backfill 12914944 12915044
-```
-
-This queues the blocks in Redis to be pulled off by the mev-inspect-worker service
-
-To increase or decrease parallelism, update the replicaCount value for the mev-inspect-workers helm chart
-
-Locally, this can be done by editing Tiltfile and changing "replicaCount=1" to your desired parallelism:
-```
-k8s_yaml(helm(
-    './k8s/mev-inspect-workers',
-    name='mev-inspect-workers',
-    set=["replicaCount=1"],
-))
-```
-
-You can see worker pods spin up then complete by watching the status of all pods
-```
-watch kubectl get pods
-```
-
-To see progress and failed batches, connect to Redis with
-```
-./mev redis
-```
-
-For total messages, query:
-```
-HLEN dramatiq:default.msgs
-```
-
-For messages failed and waiting to retry in the delay queue (DQ), query:
-```
-HGETALL dramatiq:default.DQ.msgs
-```
-
-For messages permanently failed in the dead letter queue (XQ), query:
-```
-HGETALL dramatiq:default.XQ.msgs
-```
-
-To clear the queue, delete keys for the main queue and delay queue
-```
-DEL dramatiq:default.msgs
-DEL dramatiq:default.DQ.msgs
-```
-
-For more information on queues, see the [spec shared by dramatiq](https://github.com/Bogdanp/dramatiq/blob/24cbc0dc551797783f41b08ea461e1b5d23a4058/dramatiq/brokers/redis/dispatch.lua#L24-L43)
-
-**Backfilling a list of blocks**
-
-Create a file containing a block per row, for example blocks.txt containing:
-```
-12500000
-12500001
-12500002
-```
-
-Then queue the blocks with
-```
-cat blocks.txt | ./mev block-list
-```
-
-To watch the logs for a given worker pod, take its pod name using the above, then run:
-```
-kubectl logs -f pod/mev-inspect-worker-abcdefg
-```
-
-(where `mev-inspect-worker-abcdefg` is your actual pod name)
-
-
-### Exploring
-
-All inspect output data is stored in Postgres.
-
-To connect to the local Postgres database for querying, launch a client container with:
-
-```
-./mev db
-```
-
-When you see the prompt:
-
-```
-mev_inspect=#
-```
-
-You're ready to query!
-
-Try finding the total number of swaps decoded with UniswapV3Pool:
-
-```
-SELECT COUNT(*) FROM swaps WHERE abi_name='UniswapV3Pool';
-```
-
-or top 10 arbs by gross profit that took profit in WETH:
-
-```
-SELECT *
-FROM arbitrages
-WHERE profit_token_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'
-ORDER BY profit_amount DESC
-LIMIT 10;
-```
-
-Postgres tip: Enter `\x` to enter "Explanded display" mode which looks nicer for results with many columns.
-
-## FAQ
-
-### How do I delete / reset my local postgres data?
-
-Stop the system if running:
-
-```
-tilt down
-```
-
-Delete it with:
-
-```
-kubectl delete pvc data-postgresql-postgresql-0
-```
-
-Start back up again:
-
-```
-tilt up
-```
-
-And rerun migrations to create the tables again:
-
-```
-./mev exec alembic upgrade head
-```
-
-### I was using the docker-compose setup and want to switch to kube, now what?
-
-Re-add the old `docker-compose.yml` file to your mev-inspect-py directory.
-
-A copy can be found [here](https://github.com/flashbots/mev-inspect-py/blob/ef60c097719629a7d2dc56c6e6c9a100fb706f76/docker-compose.yml)
-
-Tear down docker-compose resources:
-
-```
-docker compose down
-```
-
-Then go through the steps in the current README for kube setup.
-
-### Error from server (AlreadyExists): pods "postgres-client" already exists
-
-This means the postgres client container didn't shut down correctly.
-
-Delete this one with:
-
-```
-kubectl delete pod/postgres-client
-```
-
-Then start it back up again.
-
-## Maintainers
-
-- [@lukevs](https://github.com/lukevs)
-- [@gheise](https://github.com/gheise)
-- [@bertmiller](https://github.com/bertmiller)
+3. Add a new engine for mev_inspect with
+    - host: db
+    - user / password: see `.env`
 
 ## Contributing
 
-[Flashbots](https://flashbots.net) is a research and development collective working on mitigating the negative externalities of decentralized economies. We contribute with the larger free software community to illuminate the dark forest.
+Pre-commit is used to maintain a consistent style, prevent errors and ensure test coverage. 
 
-You are welcome here <3.
+Install pre-commit with:
+```
+poetry run pre-commit install
+```
 
-- If you want to join us, come and say hi in our [Discord chat](https://discord.gg/7hvTycdNcK).
-- If you have a question, feedback or a bug report for this project, please [open a new Issue](https://github.com/flashbots/mev-inspect-py/issues).
-- If you would like to contribute with code, check the [CONTRIBUTING file](CONTRIBUTING.md).
-- We just ask you to be nice.
-
-## Security
-
-If you find a security vulnerability on this project or any other initiative related to Flashbots, please let us know sending an email to security@flashbots.net.
-
----
-
-Made with ☀️  by the ⚡🤖 collective.
+Update README if needed

Tiltfile

@@ -1,119 +1,30 @@
-load("ext://helm_remote", "helm_remote")
-load("ext://secret", "secret_from_dict")
-load("ext://configmap", "configmap_from_dict")
+load('ext://helm_remote', 'helm_remote')
+load('ext://restart_process', 'docker_build_with_restart')
+load('ext://secret', 'secret_from_dict')
+load('ext://configmap', 'configmap_from_dict')
 
 helm_remote("postgresql",
-            repo_name="bitnami",
-            repo_url="https://charts.bitnami.com/bitnami",
-            set=["auth.postgresPassword=password", "auth.database=mev_inspect"],
-)
-
-helm_remote("redis",
-            repo_name="bitnami",
-            repo_url="https://charts.bitnami.com/bitnami",
-            set=["global.redis.password=password"],
+            repo_name='bitnami',
+            repo_url='https://charts.bitnami.com/bitnami',
+            set=["postgresqlPassword=password", "postgresqlDatabase=mev_inspect"],
 )
 
 k8s_yaml(configmap_from_dict("mev-inspect-rpc", inputs = {
     "url" : os.environ["RPC_URL"],
 }))
 
-k8s_yaml(configmap_from_dict("mev-inspect-listener-healthcheck", inputs = {
-    "url" : os.getenv("LISTENER_HEALTHCHECK_URL", default=""),
-}))
-
 k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = {
     "username" : "postgres",
     "password": "password",
-    "host": "postgresql",
 }))
 
-# if using https://github.com/taarushv/trace-db
-# k8s_yaml(secret_from_dict("trace-db-credentials", inputs = {
-#     "username" : "username",
-#     "password": "password",
-#     "host": "trace-db-postgresql",
-# }))
-
-docker_build("mev-inspect-py", ".",
+docker_build_with_restart('mev-inspect-py', '.',
+    entrypoint="/app/run.sh",
     live_update=[
-        sync(".", "/app"),
-        run("cd /app && poetry install",
-            trigger="./pyproject.toml"),
+        sync('.', '/app'),
+        run('cd /app && poetry install',
+            trigger='./pyproject.toml'),
     ],
+    platform='linux/arm64',
 )
-
-k8s_yaml(helm(
-    './k8s/mev-inspect',
-    name='mev-inspect',
-    set=[
-        "extraEnv[0].name=AWS_ACCESS_KEY_ID",
-        "extraEnv[0].value=foobar",
-        "extraEnv[1].name=AWS_SECRET_ACCESS_KEY",
-        "extraEnv[1].value=foobar",
-        "extraEnv[2].name=AWS_REGION",
-        "extraEnv[2].value=us-east-1",
-        "extraEnv[3].name=AWS_ENDPOINT_URL",
-        "extraEnv[3].value=http://localstack:4566",
-    ],
-))
-
-k8s_yaml(helm(
-    './k8s/mev-inspect-workers',
-    name='mev-inspect-workers',
-    set=[
-        "extraEnv[0].name=AWS_ACCESS_KEY_ID",
-        "extraEnv[0].value=foobar",
-        "extraEnv[1].name=AWS_SECRET_ACCESS_KEY",
-        "extraEnv[1].value=foobar",
-        "extraEnv[2].name=AWS_REGION",
-        "extraEnv[2].value=us-east-1",
-        "extraEnv[3].name=AWS_ENDPOINT_URL",
-        "extraEnv[3].value=http://localstack:4566",
-        "replicaCount=1",
-    ],
-))
-
-k8s_resource(
-    workload="mev-inspect",
-    resource_deps=["postgresql", "redis-master"],
-)
-
-k8s_resource(
-    workload="mev-inspect-workers",
-    resource_deps=["postgresql", "redis-master"],
-)
-
-# uncomment to enable price monitor
-# k8s_yaml(helm('./k8s/mev-inspect-prices', name='mev-inspect-prices'))
-# k8s_resource(workload="mev-inspect-prices", resource_deps=["postgresql"])
-
-local_resource(
-    'pg-port-forward',
-    serve_cmd='kubectl port-forward --namespace default svc/postgresql 5432:5432',
-    resource_deps=["postgresql"]
-)
-
-# if using local S3 exports
-#k8s_yaml(secret_from_dict("mev-inspect-export", inputs = {
-#    "export-bucket-name" : "local-export",
-#    "export-bucket-region": "us-east-1",
-#    "export-aws-access-key-id": "foobar",
-#    "export-aws-secret-access-key": "foobar",
-#}))
-
-#helm_remote(
-#    "localstack",
-#    repo_name="localstack-charts",
-#    repo_url="https://localstack.github.io/helm-charts",
-#)
-#
-#local_resource(
-#    'localstack-port-forward',
-#    serve_cmd='kubectl port-forward --namespace default svc/localstack 4566:4566',
-#    resource_deps=["localstack"]
-#)
-#
-#k8s_yaml(configmap_from_dict("mev-inspect-export", inputs = {
-#    "services": "s3",
-#}))
+k8s_yaml("k8s/app.yaml")

alembic/env.py

@@ -1,14 +1,16 @@
 from logging.config import fileConfig
 
-from alembic import context
-from sqlalchemy import engine_from_config, pool
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
 
-from mev_inspect.db import get_inspect_database_uri
+from alembic import context
+
+from mev_inspect.db import get_sqlalchemy_database_uri
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
-config.set_main_option("sqlalchemy.url", get_inspect_database_uri())
+config.set_main_option("sqlalchemy.url", get_sqlalchemy_database_uri())
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.

alembic/versions/04a3bb3740c3_change_miner_payments_and_transfers_.py

@@ -1,54 +0,0 @@
-"""Change miner payments and transfers primary keys to include block number
-
-Revision ID: 04a3bb3740c3
-Revises: a10d68643476
-Create Date: 2021-11-02 22:42:01.702538
-
-"""
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "04a3bb3740c3"
-down_revision = "a10d68643476"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # transfers
-    op.execute("ALTER TABLE transfers DROP CONSTRAINT transfers_pkey")
-    op.create_primary_key(
-        "transfers_pkey",
-        "transfers",
-        ["block_number", "transaction_hash", "trace_address"],
-    )
-    op.drop_index("ix_transfers_block_number")
-
-    # miner_payments
-    op.execute("ALTER TABLE miner_payments DROP CONSTRAINT miner_payments_pkey")
-    op.create_primary_key(
-        "miner_payments_pkey",
-        "miner_payments",
-        ["block_number", "transaction_hash"],
-    )
-    op.drop_index("ix_block_number")
-
-
-def downgrade():
-    # transfers
-    op.execute("ALTER TABLE transfers DROP CONSTRAINT transfers_pkey")
-    op.create_index("ix_transfers_block_number", "transfers", ["block_number"])
-    op.create_primary_key(
-        "transfers_pkey",
-        "transfers",
-        ["transaction_hash", "trace_address"],
-    )
-
-    # miner_payments
-    op.execute("ALTER TABLE miner_payments DROP CONSTRAINT miner_payments_pkey")
-    op.create_index("ix_block_number", "miner_payments", ["block_number"])
-    op.create_primary_key(
-        "miner_payments_pkey",
-        "miner_payments",
-        ["transaction_hash"],
-    )

alembic/versions/04b76ab1d2af_change_blocks_timestamp_to_timestamp.py

@@ -1,35 +0,0 @@
-"""Change blocks.timestamp to timestamp
-
-Revision ID: 04b76ab1d2af
-Revises: 2c90b2b8a80b
-Create Date: 2021-11-26 15:31:21.111693
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "04b76ab1d2af"
-down_revision = "0cef835f7b36"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.alter_column(
-        "blocks",
-        "block_timestamp",
-        type_=sa.TIMESTAMP,
-        nullable=False,
-        postgresql_using="TO_TIMESTAMP(block_timestamp)",
-    )
-
-
-def downgrade():
-    op.alter_column(
-        "blocks",
-        "block_timestamp",
-        type_=sa.Numeric,
-        nullable=False,
-        postgresql_using="extract(epoch FROM block_timestamp)",
-    )

alembic/versions/070819d86587_create_punk_snipe.py

@@ -1,34 +0,0 @@
-"""empty message
-
-Revision ID: 070819d86587
-Revises: d498bdb0a641
-Create Date: 2021-11-26 18:25:13.402822
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "d498bdb0a641"
-down_revision = "b9fa1ecc9929"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-        "punk_snipes",
-        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
-        sa.Column("block_number", sa.Numeric, nullable=False),
-        sa.Column("transaction_hash", sa.String(66), nullable=False),
-        sa.Column("trace_address", sa.String(256), nullable=False),
-        sa.Column("from_address", sa.String(256), nullable=False),
-        sa.Column("punk_index", sa.Numeric, nullable=False),
-        sa.Column("min_acceptance_price", sa.Numeric, nullable=False),
-        sa.Column("acceptance_price", sa.Numeric, nullable=False),
-        sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
-    )
-
-
-def downgrade():
-    op.drop_table("punk_snipes")

alembic/versions/083978d6e455_create_miner_payments_table.py

@@ -8,6 +8,7 @@ Create Date: 2021-08-30 17:42:25.548130
 import sqlalchemy as sa
 from alembic import op
 
+
 # revision identifiers, used by Alembic.
 revision = "083978d6e455"
 down_revision = "92f28a2b4f52"

alembic/versions/0cef835f7b36_rename_pool_address_to_contract_address.py

@@ -1,26 +0,0 @@
-"""Rename pool_address to contract_address
-
-Revision ID: 0cef835f7b36
-Revises: 5427d62a2cc0
-Create Date: 2021-11-19 15:36:15.152622
-
-"""
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "0cef835f7b36"
-down_revision = "5427d62a2cc0"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.alter_column(
-        "swaps", "pool_address", nullable=False, new_column_name="contract_address"
-    )
-
-
-def downgrade():
-    op.alter_column(
-        "swaps", "contract_address", nullable=False, new_column_name="pool_address"
-    )

alembic/versions/15ba9c27ee8a_add_nullable_transaction_position_field_.py

@@ -1,28 +0,0 @@
-"""Add nullable transaction_position field to swaps and traces
-
-Revision ID: 15ba9c27ee8a
-Revises: 04b76ab1d2af
-Create Date: 2021-12-02 18:24:18.218880
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "15ba9c27ee8a"
-down_revision = "ead7eb8283b9"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column(
-        "classified_traces",
-        sa.Column("transaction_position", sa.Numeric, nullable=True),
-    )
-    op.add_column("swaps", sa.Column("transaction_position", sa.Numeric, nullable=True))
-
-
-def downgrade():
-    op.drop_column("classified_traces", "transaction_position")
-    op.drop_column("swaps", "transaction_position")

alembic/versions/205ce02374b3_add_received_collateral_address_to_liquidations.py

@@ -1,26 +0,0 @@
-"""Add received_collateral_address to liquidations
-
-Revision ID: 205ce02374b3
-Revises: c8363617aa07
-Create Date: 2021-10-04 19:52:40.017084
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "205ce02374b3"
-down_revision = "c8363617aa07"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column(
-        "liquidations",
-        sa.Column("received_token_address", sa.String(256), nullable=True),
-    )
-
-
-def downgrade():
-    op.drop_column("liquidations", "received_token_address")

alembic/versions/2c90b2b8a80b_add_blocks_table.py

@@ -1,28 +0,0 @@
-"""Add blocks table
-
-Revision ID: 2c90b2b8a80b
-Revises: 04a3bb3740c3
-Create Date: 2021-11-17 18:29:13.065944
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "2c90b2b8a80b"
-down_revision = "04a3bb3740c3"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-        "blocks",
-        sa.Column("block_number", sa.Numeric, nullable=False),
-        sa.Column("block_timestamp", sa.Numeric, nullable=False),
-        sa.PrimaryKeyConstraint("block_number"),
-    )
-
-
-def downgrade():
-    op.drop_table("blocks")

alembic/versions/320e56b0a99f_add_index_on_block_number_for_miner_.py

@@ -7,6 +7,7 @@ Create Date: 2021-09-14 11:11:41.559137
 """
 from alembic import op
 
+
 # revision identifiers, used by Alembic.
 revision = "320e56b0a99f"
 down_revision = "a02f3f2c469f"

alembic/versions/3417f49d97b3_cahnge_swap_primary_key_to_include_.py

@@ -1,45 +0,0 @@
-"""Cahnge swap primary key to include block number
-
-Revision ID: 3417f49d97b3
-Revises: 205ce02374b3
-Create Date: 2021-11-02 20:50:32.854996
-
-"""
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "3417f49d97b3"
-down_revision = "205ce02374b3"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.execute("ALTER TABLE swaps DROP CONSTRAINT swaps_pkey CASCADE")
-    op.create_primary_key(
-        "swaps_pkey",
-        "swaps",
-        ["block_number", "transaction_hash", "trace_address"],
-    )
-    op.create_index(
-        "arbitrage_swaps_swaps_idx",
-        "arbitrage_swaps",
-        ["swap_transaction_hash", "swap_trace_address"],
-    )
-
-
-def downgrade():
-    op.drop_index("arbitrage_swaps_swaps_idx")
-    op.execute("ALTER TABLE swaps DROP CONSTRAINT swaps_pkey CASCADE")
-    op.create_primary_key(
-        "swaps_pkey",
-        "swaps",
-        ["transaction_hash", "trace_address"],
-    )
-    op.create_foreign_key(
-        "arbitrage_swaps_swaps_fkey",
-        "arbitrage_swaps",
-        "swaps",
-        ["swap_transaction_hash", "swap_trace_address"],
-        ["transaction_hash", "trace_address"],
-    )

alembic/versions/3c54832385e3_create_nft_trades_table.py

@@ -1,40 +0,0 @@
-"""Create NFT Trades table
-
-Revision ID: 3c54832385e3
-Revises: 4b9d289f2d74
-Create Date: 2021-12-19 22:50:28.936516
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "3c54832385e3"
-down_revision = "4b9d289f2d74"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-        "nft_trades",
-        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
-        sa.Column("abi_name", sa.String(1024), nullable=False),
-        sa.Column("transaction_hash", sa.String(66), nullable=False),
-        sa.Column("transaction_position", sa.Numeric, nullable=False),
-        sa.Column("block_number", sa.Numeric, nullable=False),
-        sa.Column("trace_address", sa.String(256), nullable=False),
-        sa.Column("protocol", sa.String(256), nullable=False),
-        sa.Column("error", sa.String(256), nullable=True),
-        sa.Column("seller_address", sa.String(256), nullable=False),
-        sa.Column("buyer_address", sa.String(256), nullable=False),
-        sa.Column("payment_token_address", sa.String(256), nullable=False),
-        sa.Column("payment_amount", sa.Numeric, nullable=False),
-        sa.Column("collection_address", sa.String(256), nullable=False),
-        sa.Column("token_id", sa.Numeric, nullable=False),
-        sa.PrimaryKeyConstraint("transaction_hash", "trace_address"),
-    )
-
-
-def downgrade():
-    op.drop_table("nft_trades")

alembic/versions/4b9d289f2d74_add_error_column_to_liquidations.py

@@ -1,23 +0,0 @@
-"""Add error column to liquidations
-
-Revision ID: 4b9d289f2d74
-Revises: 99d376cb93cc
-Create Date: 2021-12-23 14:54:28.406159
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "4b9d289f2d74"
-down_revision = "99d376cb93cc"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column("liquidations", sa.Column("error", sa.String(256), nullable=True))
-
-
-def downgrade():
-    op.drop_column("liquidations", "error")

alembic/versions/52d75a7e0533_add_punk_bid_acceptances.py

@@ -1,33 +0,0 @@
-"""empty message
-
-Revision ID: 52d75a7e0533
-Revises: 7cf0eeb41da0
-Create Date: 2021-11-26 20:35:58.954138
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "52d75a7e0533"
-down_revision = "7cf0eeb41da0"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-        "punk_bid_acceptances",
-        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
-        sa.Column("block_number", sa.Numeric, nullable=False),
-        sa.Column("transaction_hash", sa.String(66), nullable=False),
-        sa.Column("trace_address", sa.String(256), nullable=False),
-        sa.Column("from_address", sa.String(256), nullable=False),
-        sa.Column("punk_index", sa.Numeric, nullable=False),
-        sa.Column("min_price", sa.Numeric, nullable=False),
-        sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
-    )
-
-
-def downgrade():
-    op.drop_table("punk_bid_acceptances")

alembic/versions/5427d62a2cc0_change_transfers_trace_address_to_array.py

@@ -1,46 +0,0 @@
-"""Change transfers trace address to ARRAY
-
-Revision ID: 5427d62a2cc0
-Revises: d540242ae368
-Create Date: 2021-11-19 13:25:11.252774
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "5427d62a2cc0"
-down_revision = "d540242ae368"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.drop_constraint("transfers_pkey", "transfers")
-    op.alter_column(
-        "transfers",
-        "trace_address",
-        type_=sa.ARRAY(sa.Integer),
-        nullable=False,
-        postgresql_using="trace_address::int[]",
-    )
-    op.create_primary_key(
-        "transfers_pkey",
-        "transfers",
-        ["block_number", "transaction_hash", "trace_address"],
-    )
-
-
-def downgrade():
-    op.drop_constraint("transfers_pkey", "transfers")
-    op.alter_column(
-        "transfers",
-        "trace_address",
-        type_=sa.String(256),
-        nullable=False,
-    )
-    op.create_primary_key(
-        "transfers_pkey",
-        "transfers",
-        ["block_number", "transaction_hash", "trace_address"],
-    )

alembic/versions/5c5375de15fd_add_block_number_to_nft_trades_primary_.py

@@ -1,32 +0,0 @@
-"""Add block_number to nft_trades primary key
-
-Revision ID: 5c5375de15fd
-Revises: e616420acd18
-Create Date: 2022-01-21 15:27:57.790340
-
-"""
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "5c5375de15fd"
-down_revision = "e616420acd18"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.execute("ALTER TABLE nft_trades DROP CONSTRAINT nft_trades_pkey")
-    op.create_primary_key(
-        "nft_trades_pkey",
-        "nft_trades",
-        ["block_number", "transaction_hash", "trace_address"],
-    )
-
-
-def downgrade():
-    op.execute("ALTER TABLE nft_trades DROP CONSTRAINT nft_trades_pkey")
-    op.create_primary_key(
-        "nft_trades_pkey",
-        "nft_trades",
-        ["transaction_hash", "trace_address"],
-    )

alembic/versions/630783c18a93_make_gross_profit_nullable_on_summary.py

@@ -1,22 +0,0 @@
-"""Make gross profit nullable on summary
-
-Revision ID: 630783c18a93
-Revises: ab9a9e449ff9
-Create Date: 2022-01-19 23:09:51.816948
-
-"""
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "630783c18a93"
-down_revision = "ab9a9e449ff9"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.alter_column("mev_summary", "gross_profit_usd", nullable=True)
-
-
-def downgrade():
-    op.alter_column("mev_summary", "gross_profit_usd", nullable=False)

alembic/versions/7cf0eeb41da0_add_punk_bids.py

@@ -1,33 +0,0 @@
-"""empty message
-
-Revision ID: 7cf0eeb41da0
-Revises: d498bdb0a641
-Create Date: 2021-11-26 20:27:28.936516
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "7cf0eeb41da0"
-down_revision = "d498bdb0a641"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-        "punk_bids",
-        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
-        sa.Column("block_number", sa.Numeric, nullable=False),
-        sa.Column("transaction_hash", sa.String(66), nullable=False),
-        sa.Column("trace_address", sa.String(256), nullable=False),
-        sa.Column("from_address", sa.String(256), nullable=False),
-        sa.Column("punk_index", sa.Numeric, nullable=False),
-        sa.Column("price", sa.Numeric, nullable=False),
-        sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
-    )
-
-
-def downgrade():
-    op.drop_table("punk_bids")

alembic/versions/7eec417a4f3e_change_trace_addresses_to_array_types.py

@@ -8,6 +8,7 @@ Create Date: 2021-08-06 15:58:04.556762
 import sqlalchemy as sa
 from alembic import op
 
+
 # revision identifiers, used by Alembic.
 revision = "7eec417a4f3e"
 down_revision = "9d8c69b3dccb"

alembic/versions/92f28a2b4f52_add_error_column_to_swaps.py

@@ -8,6 +8,7 @@ Create Date: 2021-08-17 03:46:21.498821
 import sqlalchemy as sa
 from alembic import op
 
+
 # revision identifiers, used by Alembic.
 revision = "92f28a2b4f52"
 down_revision = "9b8ae51c5d56"

alembic/versions/99d376cb93cc_error.py

@@ -1,23 +0,0 @@
-"""error column
-
-Revision ID: 99d376cb93cc
-Revises: c4a7620a2d33
-Create Date: 2021-12-21 21:26:12.142484
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "99d376cb93cc"
-down_revision = "c4a7620a2d33"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column("arbitrages", sa.Column("error", sa.String(256), nullable=True))
-
-
-def downgrade():
-    op.drop_column("arbitrages", "error")

alembic/versions/9b8ae51c5d56_add_swap_arbitrage_join_table.py

@@ -8,6 +8,7 @@ Create Date: 2021-08-06 17:06:55.364516
 import sqlalchemy as sa
 from alembic import op
 
+
 # revision identifiers, used by Alembic.
 revision = "9b8ae51c5d56"
 down_revision = "7eec417a4f3e"

alembic/versions/9d8c69b3dccb_add_arbitrages_and_swap_join_table.py

@@ -8,6 +8,7 @@ Create Date: 2021-08-05 21:46:35.209199
 import sqlalchemy as sa
 from alembic import op
 
+
 # revision identifiers, used by Alembic.
 revision = "9d8c69b3dccb"
 down_revision = "2116e2f36a19"

alembic/versions/a02f3f2c469f_create_latest_block_table.py

@@ -8,6 +8,7 @@ Create Date: 2021-09-13 21:32:27.181344
 import sqlalchemy as sa
 from alembic import op
 
+
 # revision identifiers, used by Alembic.
 revision = "a02f3f2c469f"
 down_revision = "d70c08b4db6f"

alembic/versions/a10d68643476_change_classified_traces_primary_key_to_.py

@@ -1,34 +0,0 @@
-"""Change classified traces primary key to include block number
-
-Revision ID: a10d68643476
-Revises: 3417f49d97b3
-Create Date: 2021-11-02 22:03:26.312317
-
-"""
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "a10d68643476"
-down_revision = "3417f49d97b3"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.execute("ALTER TABLE classified_traces DROP CONSTRAINT classified_traces_pkey")
-    op.create_primary_key(
-        "classified_traces_pkey",
-        "classified_traces",
-        ["block_number", "transaction_hash", "trace_address"],
-    )
-    op.drop_index("i_block_number")
-
-
-def downgrade():
-    op.execute("ALTER TABLE classified_traces DROP CONSTRAINT classified_traces_pkey")
-    op.create_index("i_block_number", "classified_traces", ["block_number"])
-    op.create_primary_key(
-        "classified_traces_pkey",
-        "classified_traces",
-        ["transaction_hash", "trace_address"],
-    )

alembic/versions/ab9a9e449ff9_create_mev_summary_table.py

@@ -1,40 +0,0 @@
-"""Create mev_summary table
-
-Revision ID: ab9a9e449ff9
-Revises: b26ab0051a88
-Create Date: 2022-01-18 18:36:42.865154
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "ab9a9e449ff9"
-down_revision = "b26ab0051a88"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-        "mev_summary",
-        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
-        sa.Column("block_number", sa.Numeric, nullable=False),
-        sa.Column("block_timestamp", sa.TIMESTAMP, nullable=False),
-        sa.Column("protocol", sa.String(256), nullable=True),
-        sa.Column("transaction_hash", sa.String(66), nullable=False),
-        sa.Column("type", sa.String(256), nullable=False),
-        sa.Column("gross_profit_usd", sa.Numeric, nullable=False),
-        sa.Column("miner_payment_usd", sa.Numeric, nullable=False),
-        sa.Column("gas_used", sa.Numeric, nullable=False),
-        sa.Column("gas_price", sa.Numeric, nullable=False),
-        sa.Column("coinbase_transfer", sa.Numeric, nullable=False),
-        sa.Column("gas_price_with_coinbase_transfer", sa.Numeric, nullable=False),
-        sa.Column("miner_address", sa.String(256), nullable=False),
-        sa.Column("base_fee_per_gas", sa.Numeric, nullable=False),
-        sa.Column("error", sa.String(256), nullable=True),
-    )
-
-
-def downgrade():
-    op.drop_table("mev_summary")

alembic/versions/b26ab0051a88_add_profit_amount_column_to_sandwiches_.py

@@ -1,27 +0,0 @@
-"""add profit_amount column to sandwiches table
-
-Revision ID: b26ab0051a88
-Revises: 3c54832385e3
-Create Date: 2022-01-16 13:45:10.190969
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "b26ab0051a88"
-down_revision = "3c54832385e3"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column(
-        "sandwiches", sa.Column("profit_token_address", sa.String(256), nullable=True)
-    )
-    op.add_column("sandwiches", sa.Column("profit_amount", sa.Numeric, nullable=True))
-
-
-def downgrade():
-    op.drop_column("sandwiches", "profit_token_address")
-    op.drop_column("sandwiches", "profit_amount")

alembic/versions/b9fa1ecc9929_remove_liq_column.py

@@ -1,26 +0,0 @@
-"""Remove collateral_token_address column
-
-Revision ID: b9fa1ecc9929
-Revises: 04b76ab1d2af
-Create Date: 2021-12-01 23:32:40.574108
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "b9fa1ecc9929"
-down_revision = "04b76ab1d2af"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.drop_column("liquidations", "collateral_token_address")
-
-
-def downgrade():
-    op.add_column(
-        "liquidations",
-        sa.Column("collateral_token_address", sa.String(256), nullable=False),
-    )

alembic/versions/bba80d21c5a4_insert_tokens.py

@@ -1,40 +0,0 @@
-"""Add tokens to database
-
-Revision ID: bba80d21c5a4
-Revises: b26ab0051a88
-Create Date: 2022-01-19 22:19:59.514998
-
-"""
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "bba80d21c5a4"
-down_revision = "630783c18a93"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.execute(
-        """
-        INSERT INTO tokens (token_address,decimals) VALUES
-        ('0x514910771af9ca656af840dff83e8264ecf986ca',18),
-        ('0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',18),
-        ('0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',18),
-        ('0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e',18),
-        ('0x5d3a536e4d6dbd6114cc1ead35777bab948e3643',8),
-        ('0x2260fac5e5542a773aa44fbcfedf7c193bc2c599',8),
-        ('0x80fb784b7ed66730e8b1dbd9820afd29931aab03',18),
-        ('0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5',8),
-        ('0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48',6),
-        ('0xdac17f958d2ee523a2206206994597c13d831ec7',6),
-        ('0x6b175474e89094c44da98b954eedeac495271d0f',18),
-        ('0x0000000000085d4780b73119b644ae5ecd22b376',18),
-        ('0x39aa39c021dfbae8fac545936693ac917d5e7563',8),
-        ('0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9',18);
-        """
-    )
-
-
-def downgrade():
-    op.execute("DELETE FROM tokens")

alembic/versions/bdbb545f6c03_add_protocols_column_to_arbitrages.py

@@ -1,26 +0,0 @@
-"""Add protocols column to arbitrages
-
-Revision ID: bdbb545f6c03
-Revises: bba80d21c5a4
-Create Date: 2022-01-20 23:17:19.316008
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "bdbb545f6c03"
-down_revision = "bba80d21c5a4"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column(
-        "arbitrages",
-        sa.Column("protocols", sa.ARRAY(sa.String(256)), server_default="{}"),
-    )
-
-
-def downgrade():
-    op.drop_column("arbitrages", "protocols")

alembic/versions/c4a7620a2d33_create_tokens_table.py

@@ -1,28 +0,0 @@
-"""Create tokens table
-
-Revision ID: c4a7620a2d33
-Revises: 15ba9c27ee8a
-Create Date: 2021-12-21 19:12:33.940117
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "c4a7620a2d33"
-down_revision = "15ba9c27ee8a"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-        "tokens",
-        sa.Column("token_address", sa.String(256), nullable=False),
-        sa.Column("decimals", sa.Numeric, nullable=False),
-        sa.PrimaryKeyConstraint("token_address"),
-    )
-
-
-def downgrade():
-    op.drop_table("tokens")

alembic/versions/c5da44eb072c_add_index_for_classified_traces_block_.py

@@ -7,6 +7,7 @@ Create Date: 2021-07-30 17:37:27.335475
 """
 from alembic import op
 
+
 # revision identifiers, used by Alembic.
 revision = "c5da44eb072c"
 down_revision = "0660432b9840"

alembic/versions/c8363617aa07_create_liquidations_table.py

@@ -1,37 +0,0 @@
-"""Create liquidations table
-
-Revision ID: c8363617aa07
-Revises: cd96af55108e
-Create Date: 2021-09-29 14:00:06.857103
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "c8363617aa07"
-down_revision = "cd96af55108e"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-        "liquidations",
-        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
-        sa.Column("liquidated_user", sa.String(256), nullable=False),
-        sa.Column("liquidator_user", sa.String(256), nullable=False),
-        sa.Column("collateral_token_address", sa.String(256), nullable=False),
-        sa.Column("debt_token_address", sa.String(256), nullable=False),
-        sa.Column("debt_purchase_amount", sa.Numeric, nullable=False),
-        sa.Column("received_amount", sa.Numeric, nullable=False),
-        sa.Column("protocol", sa.String(256), nullable=True),
-        sa.Column("transaction_hash", sa.String(66), nullable=False),
-        sa.Column("trace_address", sa.String(256), nullable=False),
-        sa.Column("block_number", sa.Numeric, nullable=False),
-        sa.PrimaryKeyConstraint("transaction_hash", "trace_address"),
-    )
-
-
-def downgrade():
-    op.drop_table("liquidations")

alembic/versions/cd96af55108e_add_transfers_table.py

@@ -1,38 +0,0 @@
-"""Add transfers table
-
-Revision ID: cd96af55108e
-Revises: 5437dc68f4df
-Create Date: 2021-09-17 12:44:45.245137
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "cd96af55108e"
-down_revision = "320e56b0a99f"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-        "transfers",
-        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
-        sa.Column("block_number", sa.Numeric, nullable=False),
-        sa.Column("transaction_hash", sa.String(66), nullable=False),
-        sa.Column("trace_address", sa.String(256), nullable=False),
-        sa.Column("protocol", sa.String(256), nullable=True),
-        sa.Column("from_address", sa.String(256), nullable=False),
-        sa.Column("to_address", sa.String(256), nullable=False),
-        sa.Column("token_address", sa.String(256), nullable=False),
-        sa.Column("amount", sa.Numeric, nullable=False),
-        sa.Column("error", sa.String(256), nullable=True),
-        sa.PrimaryKeyConstraint("transaction_hash", "trace_address"),
-    )
-    op.create_index("ix_transfers_block_number", "transfers", ["block_number"])
-
-
-def downgrade():
-    op.drop_index("ix_transfers_block_number", "transfers")
-    op.drop_table("transfers")

alembic/versions/d540242ae368_create_usd_prices_table.py

@@ -1,29 +0,0 @@
-"""Create usd_prices table
-
-Revision ID: d540242ae368
-Revises: 2c90b2b8a80b
-Create Date: 2021-11-18 04:30:06.802857
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "d540242ae368"
-down_revision = "2c90b2b8a80b"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-        "prices",
-        sa.Column("timestamp", sa.TIMESTAMP),
-        sa.Column("usd_price", sa.Numeric, nullable=False),
-        sa.Column("token_address", sa.String(256), nullable=False),
-        sa.PrimaryKeyConstraint("token_address", "timestamp"),
-    )
-
-
-def downgrade():
-    op.drop_table("prices")

alembic/versions/d70c08b4db6f_add_to_address_and_from_address_to_.py

@@ -8,6 +8,7 @@ Create Date: 2021-08-30 22:10:04.186251
 import sqlalchemy as sa
 from alembic import op
 
+
 # revision identifiers, used by Alembic.
 revision = "d70c08b4db6f"
 down_revision = "083978d6e455"

alembic/versions/e616420acd18_add_protocols_column_to_mev_summary.py

@@ -1,26 +0,0 @@
-"""Add protocols column to mev_summary
-
-Revision ID: e616420acd18
-Revises: bdbb545f6c03
-Create Date: 2022-01-21 00:11:51.516459
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "e616420acd18"
-down_revision = "bdbb545f6c03"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.add_column(
-        "mev_summary",
-        sa.Column("protocols", sa.ARRAY(sa.String(256)), server_default="{}"),
-    )
-
-
-def downgrade():
-    op.drop_column("mev_summary", "protocols")

alembic/versions/ead7eb8283b9_create_sandwiches_and_sandwiched_swaps_.py

@@ -1,69 +0,0 @@
-"""Create sandwiches and sandwiched swaps tables
-
-Revision ID: ead7eb8283b9
-Revises: a5d80460f0e6
-Create Date: 2021-12-03 16:37:28.077158
-
-"""
-import sqlalchemy as sa
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "ead7eb8283b9"
-down_revision = "52d75a7e0533"
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-        "sandwiches",
-        sa.Column("id", sa.String(256), primary_key=True),
-        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
-        sa.Column("block_number", sa.Numeric, nullable=False),
-        sa.Column("sandwicher_address", sa.String(256), nullable=False),
-        sa.Column("frontrun_swap_transaction_hash", sa.String(256), nullable=False),
-        sa.Column("frontrun_swap_trace_address", sa.ARRAY(sa.Integer), nullable=False),
-        sa.Column("backrun_swap_transaction_hash", sa.String(256), nullable=False),
-        sa.Column("backrun_swap_trace_address", sa.ARRAY(sa.Integer), nullable=False),
-    )
-
-    op.create_index(
-        "ik_sandwiches_frontrun",
-        "sandwiches",
-        [
-            "block_number",
-            "frontrun_swap_transaction_hash",
-            "frontrun_swap_trace_address",
-        ],
-    )
-
-    op.create_index(
-        "ik_sandwiches_backrun",
-        "sandwiches",
-        ["block_number", "backrun_swap_transaction_hash", "backrun_swap_trace_address"],
-    )
-
-    op.create_table(
-        "sandwiched_swaps",
-        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
-        sa.Column("sandwich_id", sa.String(1024), primary_key=True),
-        sa.Column("block_number", sa.Numeric, primary_key=True),
-        sa.Column("transaction_hash", sa.String(66), primary_key=True),
-        sa.Column("trace_address", sa.ARRAY(sa.Integer), primary_key=True),
-        sa.ForeignKeyConstraint(["sandwich_id"], ["sandwiches.id"], ondelete="CASCADE"),
-    )
-
-    op.create_index(
-        "ik_sandwiched_swaps_secondary",
-        "sandwiched_swaps",
-        ["block_number", "transaction_hash", "trace_address"],
-    )
-
-
-def downgrade():
-    op.drop_index("ik_sandwiched_swaps_secondary")
-    op.drop_table("sandwiched_swaps")
-    op.drop_index("ik_sandwiches_frontrun")
-    op.drop_index("ik_sandwiches_backrun")
-    op.drop_table("sandwiches")

cli.py

@@ -1,219 +0,0 @@
-import fileinput
-import logging
-import os
-import sys
-from datetime import datetime
-
-import click
-import dramatiq
-
-from mev_inspect.concurrency import coro
-from mev_inspect.crud.prices import write_prices
-from mev_inspect.db import get_inspect_session, get_trace_session
-from mev_inspect.inspector import MEVInspector
-from mev_inspect.prices import fetch_prices, fetch_prices_range
-from mev_inspect.queue.broker import connect_broker
-from mev_inspect.queue.tasks import (
-    LOW_PRIORITY,
-    LOW_PRIORITY_QUEUE,
-    backfill_export_task,
-    inspect_many_blocks_task,
-)
-from mev_inspect.s3_export import export_block
-
-RPC_URL_ENV = "RPC_URL"
-
-logging.basicConfig(stream=sys.stdout, level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-@click.group()
-def cli():
-    pass
-
-
-@cli.command()
-@click.argument("block_number", type=int)
-@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
-@coro
-async def inspect_block_command(block_number: int, rpc: str):
-    inspect_db_session = get_inspect_session()
-    trace_db_session = get_trace_session()
-
-    inspector = MEVInspector(rpc)
-
-    await inspector.inspect_single_block(
-        inspect_db_session=inspect_db_session,
-        trace_db_session=trace_db_session,
-        block=block_number,
-    )
-
-
-@cli.command()
-@click.argument("block_number", type=int)
-@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
-@coro
-async def fetch_block_command(block_number: int, rpc: str):
-    trace_db_session = get_trace_session()
-
-    inspector = MEVInspector(rpc)
-    block = await inspector.create_from_block(
-        block_number=block_number,
-        trace_db_session=trace_db_session,
-    )
-
-    print(block.json())
-
-
-@cli.command()
-@click.argument("after_block", type=int)
-@click.argument("before_block", type=int)
-@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
-@click.option(
-    "--max-concurrency",
-    type=int,
-    help="maximum number of concurrent connections",
-    default=5,
-)
-@click.option(
-    "--request-timeout", type=int, help="timeout for requests to nodes", default=500
-)
-@coro
-async def inspect_many_blocks_command(
-    after_block: int,
-    before_block: int,
-    rpc: str,
-    max_concurrency: int,
-    request_timeout: int,
-):
-    inspect_db_session = get_inspect_session()
-    trace_db_session = get_trace_session()
-
-    inspector = MEVInspector(
-        rpc,
-        max_concurrency=max_concurrency,
-        request_timeout=request_timeout,
-    )
-    await inspector.inspect_many_blocks(
-        inspect_db_session=inspect_db_session,
-        trace_db_session=trace_db_session,
-        after_block=after_block,
-        before_block=before_block,
-    )
-
-
-@cli.command()
-def enqueue_block_list_command():
-    broker = connect_broker()
-    inspect_many_blocks_actor = dramatiq.actor(
-        inspect_many_blocks_task,
-        broker=broker,
-        queue_name=LOW_PRIORITY_QUEUE,
-        priority=LOW_PRIORITY,
-    )
-
-    for block_string in fileinput.input():
-        block = int(block_string)
-        logger.info(f"Sending {block} to {block+1}")
-        inspect_many_blocks_actor.send(block, block + 1)
-
-
-@cli.command()
-@click.argument("start_block", type=int)
-@click.argument("end_block", type=int)
-@click.argument("batch_size", type=int, default=10)
-def enqueue_many_blocks_command(start_block: int, end_block: int, batch_size: int):
-    broker = connect_broker()
-    inspect_many_blocks_actor = dramatiq.actor(
-        inspect_many_blocks_task,
-        broker=broker,
-        queue_name=LOW_PRIORITY_QUEUE,
-        priority=LOW_PRIORITY,
-    )
-
-    if start_block < end_block:
-        after_block = start_block
-        before_block = end_block
-
-        for batch_after_block in range(after_block, before_block, batch_size):
-            batch_before_block = min(batch_after_block + batch_size, before_block)
-            logger.info(f"Sending {batch_after_block} to {batch_before_block}")
-            inspect_many_blocks_actor.send(batch_after_block, batch_before_block)
-    else:
-        after_block = end_block
-        before_block = start_block
-
-        for batch_before_block in range(before_block, after_block, -1 * batch_size):
-            batch_after_block = max(batch_before_block - batch_size, after_block)
-            logger.info(f"Sending {batch_after_block} to {batch_before_block}")
-            inspect_many_blocks_actor.send(batch_after_block, batch_before_block)
-
-
-@cli.command()
-def fetch_all_prices():
-    inspect_db_session = get_inspect_session()
-
-    logger.info("Fetching prices")
-    prices = fetch_prices()
-
-    logger.info("Writing prices")
-    write_prices(inspect_db_session, prices)
-
-
-@cli.command()
-@click.argument("block_number", type=int)
-def enqueue_s3_export(block_number: int):
-    broker = connect_broker()
-    export_actor = dramatiq.actor(
-        backfill_export_task,
-        broker=broker,
-        queue_name=LOW_PRIORITY_QUEUE,
-        priority=LOW_PRIORITY,
-    )
-    logger.info(f"Sending block {block_number} export to queue")
-    export_actor.send(block_number)
-
-
-@cli.command()
-@click.argument("after_block", type=int)
-@click.argument("before_block", type=int)
-def enqueue_many_s3_exports(after_block: int, before_block: int):
-    broker = connect_broker()
-    export_actor = dramatiq.actor(
-        backfill_export_task,
-        broker=broker,
-        queue_name=LOW_PRIORITY_QUEUE,
-        priority=LOW_PRIORITY,
-    )
-    logger.info(f"Sending blocks {after_block} to {before_block} to queue")
-    for block_number in range(after_block, before_block):
-        export_actor.send(block_number)
-
-
-@cli.command()
-@click.argument("block_number", type=int)
-def s3_export(block_number: int):
-    inspect_db_session = get_inspect_session()
-    logger.info(f"Exporting {block_number}")
-    export_block(inspect_db_session, block_number)
-
-
-@cli.command()
-@click.argument("after", type=click.DateTime(formats=["%Y-%m-%d", "%m-%d-%Y"]))
-@click.argument("before", type=click.DateTime(formats=["%Y-%m-%d", "%m-%d-%Y"]))
-def fetch_range(after: datetime, before: datetime):
-    inspect_db_session = get_inspect_session()
-
-    logger.info("Fetching prices")
-    prices = fetch_prices_range(after, before)
-
-    logger.info("Writing prices")
-    write_prices(inspect_db_session, prices)
-
-
-def get_rpc_url() -> str:
-    return os.environ["RPC_URL"]
-
-
-if __name__ == "__main__":
-    cli()

docker-compose.yml

@@ -0,0 +1,24 @@
+services:
+  mev-inspect:
+    build: .
+    depends_on:
+      - db
+    env_file:
+      - .env
+    volumes:
+      - .:/app
+    tty: true
+
+  db:
+    image: postgres:12
+    volumes:
+      - mev-inspect-db-data:/var/lib/postgresql/data/pgdata
+    env_file:
+      - .env
+    environment:
+      - PGDATA=/var/lib/postgresql/data/pgdata
+    ports:
+      - 5432:5432
+
+volumes:
+  mev-inspect-db-data:

k8s/app.yaml

@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mev-inspect-deployment
+  labels:
+    app: mev-inspect
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mev-inspect
+  template:
+    metadata:
+      labels:
+        app: mev-inspect
+    spec:
+      containers:
+      - name: mev-inspect
+        image: mev-inspect-py
+        command: [ "/app/run.sh" ]
+        env:
+        - name: POSTGRES_USER
+          valueFrom:
+            secretKeyRef:
+              name: mev-inspect-db-credentials
+              key: username
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: mev-inspect-db-credentials
+              key: password
+        - name: POSTGRES_HOST
+          value: postgresql
+        - name: RPC_URL
+          valueFrom:
+            configMapKeyRef:
+              name: mev-inspect-rpc
+              key: url
+        livenessProbe:
+          exec:
+            command:
+            - ls
+            - /
+          initialDelaySeconds: 20
+          periodSeconds: 5

k8s/mev-inspect-prices/.helmignore

@@ -1,23 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

k8s/mev-inspect-prices/Chart.yaml

@@ -1,24 +0,0 @@
-apiVersion: v2
-name: mev-inspect-prices
-description: A Helm chart for Kubernetes
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
-type: application
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
-appVersion: "1.16.0"

k8s/mev-inspect-prices/templates/_helpers.tpl

@@ -1,62 +0,0 @@
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "mev-inspect-prices.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "mev-inspect-prices.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "mev-inspect-prices.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "mev-inspect-prices.labels" -}}
-helm.sh/chart: {{ include "mev-inspect-prices.chart" . }}
-{{ include "mev-inspect-prices.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Selector labels
-*/}}
-{{- define "mev-inspect-prices.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "mev-inspect-prices.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end }}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "mev-inspect-prices.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create }}
-{{- default (include "mev-inspect-prices.fullname" .) .Values.serviceAccount.name }}
-{{- else }}
-{{- default "default" .Values.serviceAccount.name }}
-{{- end }}
-{{- end }}

k8s/mev-inspect-prices/templates/cronjob.yaml

@@ -1,35 +0,0 @@
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: {{ include "mev-inspect-prices.fullname" . }}
-spec:
-  schedule: "0 */1 * * *"
-  successfulJobsHistoryLimit: 0
-  jobTemplate:
-    spec:
-      template:
-        spec:
-          containers:
-            - name: {{ .Chart.Name }}
-              image: "{{ .Values.image.repository }}"
-              imagePullPolicy: {{ .Values.image.pullPolicy }}
-              args:
-              - run
-              - fetch-all-prices
-              env:
-              - name: POSTGRES_HOST
-                valueFrom:
-                  secretKeyRef:
-                    name: mev-inspect-db-credentials
-                    key: host
-              - name: POSTGRES_USER
-                valueFrom:
-                  secretKeyRef:
-                    name: mev-inspect-db-credentials
-                    key: username
-              - name: POSTGRES_PASSWORD
-                valueFrom:
-                  secretKeyRef:
-                    name: mev-inspect-db-credentials
-                    key: password
-          restartPolicy: Never

k8s/mev-inspect-prices/values.yaml

@@ -1,7 +0,0 @@
-image:
-  repository: mev-inspect-py
-  pullPolicy: IfNotPresent
-
-imagePullSecrets: []
-nameOverride: ""
-fullnameOverride: ""

k8s/mev-inspect-workers/.helmignore

@@ -1,23 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

k8s/mev-inspect-workers/Chart.yaml

@@ -1,24 +0,0 @@
-apiVersion: v2
-name: mev-inspect-workers
-description: A Helm chart for Kubernetes
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
-type: application
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
-appVersion: "1.16.0"

k8s/mev-inspect-workers/templates/_helpers.tpl

@@ -1,62 +0,0 @@
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "mev-inspect-worker.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "mev-inspect-worker.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "mev-inspect-worker.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "mev-inspect-worker.labels" -}}
-helm.sh/chart: {{ include "mev-inspect-worker.chart" . }}
-{{ include "mev-inspect-worker.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Selector labels
-*/}}
-{{- define "mev-inspect-worker.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "mev-inspect-worker.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end }}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "mev-inspect-worker.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create }}
-{{- default (include "mev-inspect-worker.fullname" .) .Values.serviceAccount.name }}
-{{- else }}
-{{- default "default" .Values.serviceAccount.name }}
-{{- end }}
-{{- end }}

k8s/mev-inspect-workers/templates/deployment.yaml

@@ -1,133 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "mev-inspect-worker.fullname" . }}
-  labels:
-    {{- include "mev-inspect-worker.labels" . | nindent 4 }}
-spec:
-  replicas: {{ .Values.replicaCount }}
-  selector:
-    matchLabels:
-      {{- include "mev-inspect-worker.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      {{- with .Values.podAnnotations }}
-      annotations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      labels:
-        {{- include "mev-inspect-worker.selectorLabels" . | nindent 8 }}
-    spec:
-      {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      securityContext:
-        {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      containers:
-        - name: {{ .Chart.Name }}
-          securityContext:
-            {{- toYaml .Values.securityContext | nindent 12 }}
-          image: "{{ .Values.image.repository }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          args: ["run", "dramatiq", "worker", "--threads=1", "--processes=1"]
-          livenessProbe:
-            exec:
-              command:
-              - ls
-              - /
-            initialDelaySeconds: 20
-            periodSeconds: 10
-            timeoutSeconds: 5
-          resources:
-            {{- toYaml .Values.resources | nindent 12 }}
-          env:
-          - name: POSTGRES_HOST
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-db-credentials
-                key: host
-          - name: POSTGRES_USER
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-db-credentials
-                key: username
-          - name: POSTGRES_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-db-credentials
-                key: password
-          - name: REDIS_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: redis
-                key: redis-password
-          - name: TRACE_DB_HOST
-            valueFrom:
-              secretKeyRef:
-                name: trace-db-credentials
-                key: host
-                optional: true
-          - name: TRACE_DB_USER
-            valueFrom:
-              secretKeyRef:
-                name: trace-db-credentials
-                key: username
-                optional: true
-          - name: TRACE_DB_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: trace-db-credentials
-                key: password
-                optional: true
-          - name: RPC_URL
-            valueFrom:
-              configMapKeyRef:
-                name: mev-inspect-rpc
-                key: url
-          - name: LISTENER_HEALTHCHECK_URL
-            valueFrom:
-              configMapKeyRef:
-                name: mev-inspect-listener-healthcheck
-                key: url
-                optional: true
-          - name: EXPORT_BUCKET_NAME
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-export
-                key: export-bucket-name
-                optional: true
-          - name: EXPORT_BUCKET_REGION
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-export
-                key: export-bucket-region
-                optional: true
-          - name: EXPORT_AWS_ACCESS_KEY_ID
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-export
-                key: export-aws-access-key-id
-                optional: true
-          - name: EXPORT_AWS_SECRET_ACCESS_KEY
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-export
-                key: export-aws-secret-access-key
-                optional: true
-          {{- range .Values.extraEnv }}
-          - name: {{ .name }}
-            value: {{ .value }}
-          {{- end }} 
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}

k8s/mev-inspect-workers/values.yaml

@@ -1,45 +0,0 @@
-# Default values for mev-inspect-workers
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-replicaCount: 1
-
-image:
-  repository: mev-inspect-py:latest
-  pullPolicy: IfNotPresent
-
-imagePullSecrets: []
-nameOverride: ""
-fullnameOverride: ""
-
-podAnnotations: {}
-
-podSecurityContext: {}
-  # fsGroup: 2000
-
-securityContext:
-  allowPrivilegeEscalation: false
-  capabilities:
-    drop:
-    - ALL
-  # readOnlyRootFilesystem: true
-  runAsNonRoot: true
-  runAsUser: 1000
-
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
-
-nodeSelector: {}
-
-tolerations: []
-
-affinity: {}

k8s/mev-inspect/.helmignore

@@ -1,23 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

k8s/mev-inspect/Chart.yaml

@@ -1,24 +0,0 @@
-apiVersion: v2
-name: mev-inspect
-description: A Helm chart for Kubernetes
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
-type: application
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
-appVersion: "1.16.0"

k8s/mev-inspect/templates/_helpers.tpl

@@ -1,62 +0,0 @@
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "mev-inspect.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "mev-inspect.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "mev-inspect.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "mev-inspect.labels" -}}
-helm.sh/chart: {{ include "mev-inspect.chart" . }}
-{{ include "mev-inspect.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
-
-{{/*
-Selector labels
-*/}}
-{{- define "mev-inspect.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "mev-inspect.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end }}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "mev-inspect.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create }}
-{{- default (include "mev-inspect.fullname" .) .Values.serviceAccount.name }}
-{{- else }}
-{{- default "default" .Values.serviceAccount.name }}
-{{- end }}
-{{- end }}

k8s/mev-inspect/templates/deployment.yaml

@@ -1,133 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "mev-inspect.fullname" . }}
-  labels:
-    {{- include "mev-inspect.labels" . | nindent 4 }}
-spec:
-  replicas: {{ .Values.replicaCount }}
-  selector:
-    matchLabels:
-      {{- include "mev-inspect.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      {{- with .Values.podAnnotations }}
-      annotations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      labels:
-        {{- include "mev-inspect.selectorLabels" . | nindent 8 }}
-    spec:
-      {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      securityContext:
-        {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      containers:
-        - name: {{ .Chart.Name }}
-          securityContext:
-            {{- toYaml .Values.securityContext | nindent 12 }}
-          image: "{{ .Values.image.repository }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          args: ["run", "python", "loop.py"]
-          livenessProbe:
-            exec:
-              command:
-              - ls
-              - /
-            initialDelaySeconds: 20
-            periodSeconds: 10
-            timeoutSeconds: 5
-          resources:
-            {{- toYaml .Values.resources | nindent 12 }}
-          env:
-          - name: POSTGRES_HOST
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-db-credentials
-                key: host
-          - name: POSTGRES_USER
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-db-credentials
-                key: username
-          - name: POSTGRES_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-db-credentials
-                key: password
-          - name: REDIS_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: redis
-                key: redis-password
-          - name: TRACE_DB_HOST
-            valueFrom:
-              secretKeyRef:
-                name: trace-db-credentials
-                key: host
-                optional: true
-          - name: TRACE_DB_USER
-            valueFrom:
-              secretKeyRef:
-                name: trace-db-credentials
-                key: username
-                optional: true
-          - name: TRACE_DB_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: trace-db-credentials
-                key: password
-                optional: true
-          - name: RPC_URL
-            valueFrom:
-              configMapKeyRef:
-                name: mev-inspect-rpc
-                key: url
-          - name: LISTENER_HEALTHCHECK_URL
-            valueFrom:
-              configMapKeyRef:
-                name: mev-inspect-listener-healthcheck
-                key: url
-                optional: true
-          - name: EXPORT_BUCKET_NAME
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-export
-                key: export-bucket-name
-                optional: true
-          - name: EXPORT_BUCKET_REGION
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-export
-                key: export-bucket-region
-                optional: true
-          - name: EXPORT_AWS_ACCESS_KEY_ID
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-export
-                key: export-aws-access-key-id
-                optional: true
-          - name: EXPORT_AWS_SECRET_ACCESS_KEY
-            valueFrom:
-              secretKeyRef:
-                name: mev-inspect-export
-                key: export-aws-secret-access-key
-                optional: true
-          {{- range .Values.extraEnv }}
-          - name: {{ .name }}
-            value: {{ .value }}
-          {{- end }}
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}

k8s/mev-inspect/values.yaml

@@ -1,46 +0,0 @@
-# Default values for mev-inspect.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-replicaCount: 1
-
-image:
-  repository: mev-inspect-py:latest
-  pullPolicy: IfNotPresent
-
-imagePullSecrets: []
-nameOverride: ""
-fullnameOverride: ""
-
-podAnnotations: {}
-
-podSecurityContext: {}
-  # fsGroup: 2000
-
-securityContext:
-  allowPrivilegeEscalation: false
-  capabilities:
-    drop:
-      - all
-  #readOnlyRootFilesystem: true
-  runAsNonRoot: true
-  runAsUser: 1000
-
-
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
-
-nodeSelector: {}
-
-tolerations: []
-
-affinity: {}

listener

@@ -1,54 +0,0 @@
-#!/bin/sh
-
-set -e
-
-NAME=listener
-PIDFILE=/home/flashbot/$NAME.pid
-DAEMON=/bin/bash
-DAEMON_OPTS='-c "poetry run python listener.py"'
-
-case "$1" in
-  start)
-        echo -n "Starting daemon: "$NAME
-	start-stop-daemon \
-        --background \
-        --chdir /app \
-        --chuid flashbot \
-        --start \
-        --quiet \
-        --pidfile $PIDFILE \
-        --make-pidfile \
-        --startas /bin/bash -- -c "poetry run python listener.py"
-        echo "."
-	;;
-  stop)
-        echo -n "Stopping daemon: "$NAME
-	start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE
-    rm $PIDFILE
-        echo "."
-	;;
-  tail)
-        tail -f listener.log
-	;;
-  restart)
-        echo -n "Restarting daemon: "$NAME
-	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile $PIDFILE
-    rm $PIDFILE
-	start-stop-daemon \
-        --background \
-        --chdir /app \
-        --chuid flashbot \
-        --start \
-        --quiet \
-        --pidfile $PIDFILE \
-        --make-pidfile \
-        --startas /bin/bash -- -c "poetry run python listener.py"
-        echo "."
-    ;;
-
-  *)
-	echo "Usage: "$1" {start|stop|restart|tail}"
-	exit 1
-esac
-
-exit 0

listener.py

@@ -1,126 +0,0 @@
-import asyncio
-import logging
-import os
-
-import dramatiq
-from aiohttp_retry import ExponentialRetry, RetryClient
-
-from mev_inspect.block import get_latest_block_number
-from mev_inspect.concurrency import coro
-from mev_inspect.crud.latest_block_update import (
-    find_latest_block_update,
-    update_latest_block,
-)
-from mev_inspect.db import get_inspect_session, get_trace_session
-from mev_inspect.inspector import MEVInspector
-from mev_inspect.provider import get_base_provider
-from mev_inspect.queue.broker import connect_broker
-from mev_inspect.queue.tasks import (
-    HIGH_PRIORITY,
-    HIGH_PRIORITY_QUEUE,
-    realtime_export_task,
-)
-from mev_inspect.signal_handler import GracefulKiller
-
-logging.basicConfig(filename="listener.log", filemode="a", level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-# lag to make sure the blocks we see are settled
-BLOCK_NUMBER_LAG = 5
-
-
-@coro
-async def run():
-    rpc = os.getenv("RPC_URL")
-    if rpc is None:
-        raise RuntimeError("Missing environment variable RPC_URL")
-
-    healthcheck_url = os.getenv("LISTENER_HEALTHCHECK_URL")
-
-    logger.info("Starting...")
-
-    killer = GracefulKiller()
-
-    inspect_db_session = get_inspect_session()
-    trace_db_session = get_trace_session()
-
-    broker = connect_broker()
-    export_actor = dramatiq.actor(
-        realtime_export_task,
-        broker=broker,
-        queue_name=HIGH_PRIORITY_QUEUE,
-        priority=HIGH_PRIORITY,
-    )
-
-    inspector = MEVInspector(rpc)
-    base_provider = get_base_provider(rpc)
-
-    while not killer.kill_now:
-        await inspect_next_block(
-            inspector,
-            inspect_db_session,
-            trace_db_session,
-            base_provider,
-            healthcheck_url,
-            export_actor,
-        )
-
-    logger.info("Stopping...")
-
-
-async def inspect_next_block(
-    inspector: MEVInspector,
-    inspect_db_session,
-    trace_db_session,
-    base_provider,
-    healthcheck_url,
-    export_actor,
-):
-
-    latest_block_number = await get_latest_block_number(base_provider)
-    last_written_block = find_latest_block_update(inspect_db_session)
-
-    logger.info(f"Latest block: {latest_block_number}")
-    logger.info(f"Last written block: {last_written_block}")
-
-    if last_written_block is None:
-        # maintain lag if no blocks written yet
-        last_written_block = latest_block_number - BLOCK_NUMBER_LAG - 1
-
-    if last_written_block < (latest_block_number - BLOCK_NUMBER_LAG):
-        block_number = last_written_block + 1
-
-        logger.info(f"Writing block: {block_number}")
-
-        await inspector.inspect_single_block(
-            inspect_db_session=inspect_db_session,
-            trace_db_session=trace_db_session,
-            block=block_number,
-        )
-
-        update_latest_block(inspect_db_session, block_number)
-
-        logger.info(f"Sending block {block_number} for export")
-        export_actor.send(block_number)
-
-        if healthcheck_url:
-            await ping_healthcheck_url(healthcheck_url)
-    else:
-        await asyncio.sleep(5)
-
-
-async def ping_healthcheck_url(url):
-    retry_options = ExponentialRetry(attempts=3)
-
-    async with RetryClient(
-        raise_for_status=False, retry_options=retry_options
-    ) as client:
-        async with client.get(url) as _response:
-            pass
-
-
-if __name__ == "__main__":
-    try:
-        run()
-    except Exception as e:
-        logger.error(e)

loop.py

@@ -1,21 +0,0 @@
-import logging
-import time
-
-from mev_inspect.signal_handler import GracefulKiller
-
-logging.basicConfig(filename="loop.log", level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-def run():
-    logger.info("Starting...")
-
-    killer = GracefulKiller()
-    while not killer.kill_now:
-        time.sleep(1)
-
-    logger.info("Stopping...")
-
-
-if __name__ == "__main__":
-    run()

mev

@@ -1,129 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-DB_NAME=mev_inspect
-
-function get_kube_secret(){
-    kubectl get secrets $1 -o jsonpath="{.data.$2}" | base64 --decode
-}
-
-function get_kube_db_secret(){
-    kubectl get secrets mev-inspect-db-credentials -o jsonpath="{.data.$1}" | base64 --decode
-}
-
-function db(){
-    host=$(get_kube_secret "mev-inspect-db-credentials" "host")
-    username=$(get_kube_secret "mev-inspect-db-credentials" "username")
-    password=$(get_kube_secret "mev-inspect-db-credentials" "password")
-
-    kubectl run -i --rm --tty postgres-client-$RANDOM \
-        --env="PGPASSWORD=$password"  \
-        --image=jbergknoff/postgresql-client \
-        -- $DB_NAME --host=$host --user=$username
-}
-
-function redis(){
-    echo "To continue, enter 'shift + r'"
-    redis_password=$(get_kube_secret "redis" "redis-password")
-    kubectl run -i --rm --tty \
-        --namespace default redis-client-$RANDOM \
-        --env REDIS_PASSWORD=$redis_password \
-        --image docker.io/bitnami/redis:6.2.6-debian-10-r0 \
-        --command -- redis-cli -h redis-master -a $redis_password
-}
-
-case "$1" in
-  db)
-        echo "Connecting to $DB_NAME"
-        db
-	;;
-  redis)
-        echo "Connecting to redis"
-        redis
-    ;;
-  listener)
-        kubectl exec -ti deploy/mev-inspect -- ./listener $2
-	;;
-  block-list)
-        echo "Backfilling blocks from stdin"
-        kubectl exec -i deploy/mev-inspect -- poetry run enqueue-block-list
-	;;
-  backfill)
-        after_block_number=$2
-        before_block_number=$3
-
-        echo "Backfilling from $after_block_number to $before_block_number"
-        kubectl exec -ti deploy/mev-inspect -- poetry run enqueue-many-blocks $after_block_number $before_block_number
-	;;
-  inspect)
-        block_number=$2
-        echo "Inspecting block $block_number"
-        kubectl exec -ti deploy/mev-inspect -- poetry run inspect-block $block_number
-	;;
-  inspect-many)
-        after_block_number=$2
-        before_block_number=$3
-        echo "Inspecting from block $after_block_number to $before_block_number"
-        kubectl exec -ti deploy/mev-inspect -- \
-            poetry run inspect-many-blocks $after_block_number $before_block_number
-	;;
-  test)
-        shift
-        echo "Running tests"
-        kubectl exec -ti deploy/mev-inspect -- poetry run pytest tests $@
-	;;
-  fetch)
-      block_number=$2
-      echo "Fetching block $block_number"
-      kubectl exec -ti deploy/mev-inspect -- poetry run fetch-block $block_number
-  ;;
-  prices)
-        shift
-        case "$1" in
-          fetch-all)
-                echo "Running price fetch-all"
-                kubectl exec -ti deploy/mev-inspect -- \
-                    poetry run fetch-all-prices
-            ;;
-          fetch-range)
-                after=$2
-                before=$3
-                echo "Running price fetch-range"
-                kubectl exec -ti deploy/mev-inspect -- \
-                    poetry run fetch-range $after $before
-            ;;
-            *)
-            echo "prices usage: "$1" {fetch-all}"
-            exit 1
-        esac
-  ;;
-  backfill-export)
-        after_block=$2
-        before_block=$3
-
-        echo "Sending $after_block to $before_block export to queue"
-        kubectl exec -ti deploy/mev-inspect -- poetry run enqueue-many-s3-exports $after_block $before_block
-  ;;
-  enqueue-s3-export)
-        block_number=$2
-
-        echo "Sending $block_number export to queue"
-        kubectl exec -ti deploy/mev-inspect -- poetry run enqueue-s3-export $block_number
-  ;;
-  s3-export)
-        block_number=$2
-
-        echo "Exporting $block_number"
-        kubectl exec -ti deploy/mev-inspect -- poetry run s3-export $block_number
-  ;;
-  exec)
-        shift
-        kubectl exec -ti deploy/mev-inspect -- $@
-  ;;
-  *)
-	echo "Usage: "$1" {db|backfill|inspect|test}"
-	exit 1
-esac
-
-exit 0

mev_inspect/abi.py

@@ -4,39 +4,23 @@ from typing import Optional
 
 from pydantic import parse_obj_as
 
-from mev_inspect.schemas.abi import ABI
-from mev_inspect.schemas.traces import Protocol
+from mev_inspect.schemas import ABI
+from mev_inspect.schemas.classified_traces import Protocol
+
 
 THIS_FILE_DIRECTORY = Path(__file__).parents[0]
 ABI_DIRECTORY_PATH = THIS_FILE_DIRECTORY / "abis"
 
 
-def get_abi_path(abi_name: str, protocol: Optional[Protocol]) -> Optional[Path]:
+def get_abi(abi_name: str, protocol: Optional[Protocol]) -> Optional[ABI]:
     abi_filename = f"{abi_name}.json"
     abi_path = (
         ABI_DIRECTORY_PATH / abi_filename
         if protocol is None
         else ABI_DIRECTORY_PATH / protocol.value / abi_filename
     )
+
     if abi_path.is_file():
-        return abi_path
-
-    return None
-
-
-# raw abi, for instantiating contract for queries (as opposed to classification, see below)
-def get_raw_abi(abi_name: str, protocol: Optional[Protocol]) -> Optional[str]:
-    abi_path = get_abi_path(abi_name, protocol)
-    if abi_path is not None:
-        with abi_path.open() as abi_file:
-            return abi_file.read()
-
-    return None
-
-
-def get_abi(abi_name: str, protocol: Optional[Protocol]) -> Optional[ABI]:
-    abi_path = get_abi_path(abi_name, protocol)
-    if abi_path is not None:
         with abi_path.open() as abi_file:
             abi_json = json.load(abi_file)
             return parse_obj_as(ABI, abi_json)

mev_inspect/abis/aave/aTokens.json

@@ -1,615 +0,0 @@
-[
-  {
-    "anonymous": false,
-    "inputs": [
-      {
-        "indexed": true,
-        "internalType": "address",
-        "name": "owner",
-        "type": "address"
-      },
-      {
-        "indexed": true,
-        "internalType": "address",
-        "name": "spender",
-        "type": "address"
-      },
-      {
-        "indexed": false,
-        "internalType": "uint256",
-        "name": "value",
-        "type": "uint256"
-      }
-    ],
-    "name": "Approval",
-    "type": "event"
-  },
-  {
-    "anonymous": false,
-    "inputs": [
-      {
-        "indexed": true,
-        "internalType": "address",
-        "name": "from",
-        "type": "address"
-      },
-      {
-        "indexed": true,
-        "internalType": "address",
-        "name": "to",
-        "type": "address"
-      },
-      {
-        "indexed": false,
-        "internalType": "uint256",
-        "name": "value",
-        "type": "uint256"
-      },
-      {
-        "indexed": false,
-        "internalType": "uint256",
-        "name": "index",
-        "type": "uint256"
-      }
-    ],
-    "name": "BalanceTransfer",
-    "type": "event"
-  },
-  {
-    "anonymous": false,
-    "inputs": [
-      {
-        "indexed": true,
-        "internalType": "address",
-        "name": "from",
-        "type": "address"
-      },
-      {
-        "indexed": true,
-        "internalType": "address",
-        "name": "target",
-        "type": "address"
-      },
-      {
-        "indexed": false,
-        "internalType": "uint256",
-        "name": "value",
-        "type": "uint256"
-      },
-      {
-        "indexed": false,
-        "internalType": "uint256",
-        "name": "index",
-        "type": "uint256"
-      }
-    ],
-    "name": "Burn",
-    "type": "event"
-  },
-  {
-    "anonymous": false,
-    "inputs": [
-      {
-        "indexed": true,
-        "internalType": "address",
-        "name": "underlyingAsset",
-        "type": "address"
-      },
-      {
-        "indexed": true,
-        "internalType": "address",
-        "name": "pool",
-        "type": "address"
-      },
-      {
-        "indexed": false,
-        "internalType": "address",
-        "name": "treasury",
-        "type": "address"
-      },
-      {
-        "indexed": false,
-        "internalType": "address",
-        "name": "incentivesController",
-        "type": "address"
-      },
-      {
-        "indexed": false,
-        "internalType": "uint8",
-        "name": "aTokenDecimals",
-        "type": "uint8"
-      },
-      {
-        "indexed": false,
-        "internalType": "string",
-        "name": "aTokenName",
-        "type": "string"
-      },
-      {
-        "indexed": false,
-        "internalType": "string",
-        "name": "aTokenSymbol",
-        "type": "string"
-      },
-      {
-        "indexed": false,
-        "internalType": "bytes",
-        "name": "params",
-        "type": "bytes"
-      }
-    ],
-    "name": "Initialized",
-    "type": "event"
-  },
-  {
-    "anonymous": false,
-    "inputs": [
-      {
-        "indexed": true,
-        "internalType": "address",
-        "name": "from",
-        "type": "address"
-      },
-      {
-        "indexed": false,
-        "internalType": "uint256",
-        "name": "value",
-        "type": "uint256"
-      },
-      {
-        "indexed": false,
-        "internalType": "uint256",
-        "name": "index",
-        "type": "uint256"
-      }
-    ],
-    "name": "Mint",
-    "type": "event"
-  },
-  {
-    "anonymous": false,
-    "inputs": [
-      {
-        "indexed": true,
-        "internalType": "address",
-        "name": "from",
-        "type": "address"
-      },
-      {
-        "indexed": true,
-        "internalType": "address",
-        "name": "to",
-        "type": "address"
-      },
-      {
-        "indexed": false,
-        "internalType": "uint256",
-        "name": "value",
-        "type": "uint256"
-      }
-    ],
-    "name": "Transfer",
-    "type": "event"
-  },
-  {
-    "inputs": [
-      
-    ],
-    "name": "UNDERLYING_ASSET_ADDRESS",
-    "outputs": [
-      {
-        "internalType": "address",
-        "name": "",
-        "type": "address"
-      }
-    ],
-    "stateMutability": "view",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "address",
-        "name": "owner",
-        "type": "address"
-      },
-      {
-        "internalType": "address",
-        "name": "spender",
-        "type": "address"
-      }
-    ],
-    "name": "allowance",
-    "outputs": [
-      {
-        "internalType": "uint256",
-        "name": "",
-        "type": "uint256"
-      }
-    ],
-    "stateMutability": "view",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "address",
-        "name": "spender",
-        "type": "address"
-      },
-      {
-        "internalType": "uint256",
-        "name": "amount",
-        "type": "uint256"
-      }
-    ],
-    "name": "approve",
-    "outputs": [
-      {
-        "internalType": "bool",
-        "name": "",
-        "type": "bool"
-      }
-    ],
-    "stateMutability": "nonpayable",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "address",
-        "name": "account",
-        "type": "address"
-      }
-    ],
-    "name": "balanceOf",
-    "outputs": [
-      {
-        "internalType": "uint256",
-        "name": "",
-        "type": "uint256"
-      }
-    ],
-    "stateMutability": "view",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "address",
-        "name": "user",
-        "type": "address"
-      },
-      {
-        "internalType": "address",
-        "name": "receiverOfUnderlying",
-        "type": "address"
-      },
-      {
-        "internalType": "uint256",
-        "name": "amount",
-        "type": "uint256"
-      },
-      {
-        "internalType": "uint256",
-        "name": "index",
-        "type": "uint256"
-      }
-    ],
-    "name": "burn",
-    "outputs": [
-      
-    ],
-    "stateMutability": "nonpayable",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      
-    ],
-    "name": "getIncentivesController",
-    "outputs": [
-      {
-        "internalType": "contract IAaveIncentivesController",
-        "name": "",
-        "type": "address"
-      }
-    ],
-    "stateMutability": "view",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "address",
-        "name": "user",
-        "type": "address"
-      }
-    ],
-    "name": "getScaledUserBalanceAndSupply",
-    "outputs": [
-      {
-        "internalType": "uint256",
-        "name": "",
-        "type": "uint256"
-      },
-      {
-        "internalType": "uint256",
-        "name": "",
-        "type": "uint256"
-      }
-    ],
-    "stateMutability": "view",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "address",
-        "name": "user",
-        "type": "address"
-      },
-      {
-        "internalType": "uint256",
-        "name": "amount",
-        "type": "uint256"
-      }
-    ],
-    "name": "handleRepayment",
-    "outputs": [
-      
-    ],
-    "stateMutability": "nonpayable",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "contract ILendingPool",
-        "name": "pool",
-        "type": "address"
-      },
-      {
-        "internalType": "address",
-        "name": "treasury",
-        "type": "address"
-      },
-      {
-        "internalType": "address",
-        "name": "underlyingAsset",
-        "type": "address"
-      },
-      {
-        "internalType": "contract IAaveIncentivesController",
-        "name": "incentivesController",
-        "type": "address"
-      },
-      {
-        "internalType": "uint8",
-        "name": "aTokenDecimals",
-        "type": "uint8"
-      },
-      {
-        "internalType": "string",
-        "name": "aTokenName",
-        "type": "string"
-      },
-      {
-        "internalType": "string",
-        "name": "aTokenSymbol",
-        "type": "string"
-      },
-      {
-        "internalType": "bytes",
-        "name": "params",
-        "type": "bytes"
-      }
-    ],
-    "name": "initialize",
-    "outputs": [
-      
-    ],
-    "stateMutability": "nonpayable",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "address",
-        "name": "user",
-        "type": "address"
-      },
-      {
-        "internalType": "uint256",
-        "name": "amount",
-        "type": "uint256"
-      },
-      {
-        "internalType": "uint256",
-        "name": "index",
-        "type": "uint256"
-      }
-    ],
-    "name": "mint",
-    "outputs": [
-      {
-        "internalType": "bool",
-        "name": "",
-        "type": "bool"
-      }
-    ],
-    "stateMutability": "nonpayable",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "uint256",
-        "name": "amount",
-        "type": "uint256"
-      },
-      {
-        "internalType": "uint256",
-        "name": "index",
-        "type": "uint256"
-      }
-    ],
-    "name": "mintToTreasury",
-    "outputs": [
-      
-    ],
-    "stateMutability": "nonpayable",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "address",
-        "name": "user",
-        "type": "address"
-      }
-    ],
-    "name": "scaledBalanceOf",
-    "outputs": [
-      {
-        "internalType": "uint256",
-        "name": "",
-        "type": "uint256"
-      }
-    ],
-    "stateMutability": "view",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      
-    ],
-    "name": "scaledTotalSupply",
-    "outputs": [
-      {
-        "internalType": "uint256",
-        "name": "",
-        "type": "uint256"
-      }
-    ],
-    "stateMutability": "view",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      
-    ],
-    "name": "totalSupply",
-    "outputs": [
-      {
-        "internalType": "uint256",
-        "name": "",
-        "type": "uint256"
-      }
-    ],
-    "stateMutability": "view",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "address",
-        "name": "recipient",
-        "type": "address"
-      },
-      {
-        "internalType": "uint256",
-        "name": "amount",
-        "type": "uint256"
-      }
-    ],
-    "name": "transfer",
-    "outputs": [
-      {
-        "internalType": "bool",
-        "name": "",
-        "type": "bool"
-      }
-    ],
-    "stateMutability": "nonpayable",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "address",
-        "name": "sender",
-        "type": "address"
-      },
-      {
-        "internalType": "address",
-        "name": "recipient",
-        "type": "address"
-      },
-      {
-        "internalType": "uint256",
-        "name": "amount",
-        "type": "uint256"
-      }
-    ],
-    "name": "transferFrom",
-    "outputs": [
-      {
-        "internalType": "bool",
-        "name": "",
-        "type": "bool"
-      }
-    ],
-    "stateMutability": "nonpayable",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "address",
-        "name": "from",
-        "type": "address"
-      },
-      {
-        "internalType": "address",
-        "name": "to",
-        "type": "address"
-      },
-      {
-        "internalType": "uint256",
-        "name": "value",
-        "type": "uint256"
-      }
-    ],
-    "name": "transferOnLiquidation",
-    "outputs": [
-      
-    ],
-    "stateMutability": "nonpayable",
-    "type": "function"
-  },
-  {
-    "inputs": [
-      {
-        "internalType": "address",
-        "name": "user",
-        "type": "address"
-      },
-      {
-        "internalType": "uint256",
-        "name": "amount",
-        "type": "uint256"
-      }
-    ],
-    "name": "transferUnderlyingTo",
-    "outputs": [
-      {
-        "internalType": "uint256",
-        "name": "",
-        "type": "uint256"
-      }
-    ],
-    "stateMutability": "nonpayable",
-    "type": "function"
-  }
-]

mev_inspect/arbitrages.py

@@ -1,11 +1,8 @@
 from itertools import groupby
-from typing import List, Optional, Tuple
+from typing import List, Optional
 
 from mev_inspect.schemas.arbitrages import Arbitrage
 from mev_inspect.schemas.swaps import Swap
-from mev_inspect.utils import equal_within_percent
-
-MAX_TOKEN_AMOUNT_PERCENT_DIFFERENCE = 0.01
 
 
 def get_arbitrages(swaps: List[Swap]) -> List[Arbitrage]:
@@ -26,168 +23,70 @@ def get_arbitrages(swaps: List[Swap]) -> List[Arbitrage]:
 
 
 def _get_arbitrages_from_swaps(swaps: List[Swap]) -> List[Arbitrage]:
-    """
-    An arbitrage is defined as multiple swaps in a series that result in the initial token being returned
-    to the initial sender address.
-
-    There are 2 types of swaps that are most common (99%+).
-    Case I (fully routed):
-    BOT -> A/B -> B/C -> C/A -> BOT
-
-    Case II (always return to bot):
-    BOT -> A/B -> BOT -> B/C -> BOT -> A/C -> BOT
-
-    There is only 1 correct way to route Case I, but for Case II the following valid routes could be found:
-    A->B->C->A / B->C->A->B / C->A->B->C. Thus when multiple valid routes are found we filter to the set that
-    happen in valid order.
-    """
+    pool_addresses = {swap.pool_address for swap in swaps}
 
     all_arbitrages = []
 
-    start_ends = _get_all_start_end_swaps(swaps)
-    if len(start_ends) == 0:
-        return []
+    for index, first_swap in enumerate(swaps):
+        other_swaps = swaps[:index] + swaps[index + 1 :]
 
-    used_swaps: List[Swap] = []
+        if first_swap.from_address not in pool_addresses:
+            arbitrage = _get_arbitrage_starting_with_swap(first_swap, other_swaps)
 
-    for (start, ends) in start_ends:
-        if start in used_swaps:
-            continue
+            if arbitrage is not None:
+                all_arbitrages.append(arbitrage)
 
-        unused_ends = [end for end in ends if end not in used_swaps]
-        route = _get_shortest_route(start, unused_ends, swaps)
+    return all_arbitrages
 
-        if route is not None:
-            start_amount = route[0].token_in_amount
-            end_amount = route[-1].token_out_amount
+
+def _get_arbitrage_starting_with_swap(
+    start_swap: Swap,
+    other_swaps: List[Swap],
+) -> Optional[Arbitrage]:
+    swap_path = [start_swap]
+    current_swap: Swap = start_swap
+
+    while True:
+        next_swap = _get_swap_from_address(
+            current_swap.to_address,
+            current_swap.token_out_address,
+            other_swaps,
+        )
+
+        if next_swap is None:
+            return None
+
+        swap_path.append(next_swap)
+        current_swap = next_swap
+
+        if (
+            current_swap.to_address == start_swap.from_address
+            and current_swap.token_out_address == start_swap.token_in_address
+        ):
+
+            start_amount = start_swap.token_in_amount
+            end_amount = current_swap.token_out_amount
             profit_amount = end_amount - start_amount
-            error = None
-            for swap in route:
-                if swap.error is not None:
-                    error = swap.error
 
-            arb = Arbitrage(
-                swaps=route,
-                block_number=route[0].block_number,
-                transaction_hash=route[0].transaction_hash,
-                account_address=route[0].from_address,
-                profit_token_address=route[0].token_in_address,
+            return Arbitrage(
+                swaps=swap_path,
+                block_number=start_swap.block_number,
+                transaction_hash=start_swap.transaction_hash,
+                account_address=start_swap.from_address,
+                profit_token_address=start_swap.token_in_address,
                 start_amount=start_amount,
                 end_amount=end_amount,
                 profit_amount=profit_amount,
-                error=error,
             )
 
-            all_arbitrages.append(arb)
-            used_swaps.extend(route)
-
-    if len(all_arbitrages) == 1:
-        return all_arbitrages
-    else:
-        return [
-            arb
-            for arb in all_arbitrages
-            if (arb.swaps[0].trace_address < arb.swaps[-1].trace_address)
-        ]
+    return None
 
 
-def _get_shortest_route(
-    start_swap: Swap,
-    end_swaps: List[Swap],
-    all_swaps: List[Swap],
-    max_route_length: Optional[int] = None,
-) -> Optional[List[Swap]]:
-    if len(end_swaps) == 0:
-        return None
+def _get_swap_from_address(
+    address: str, token_address: str, swaps: List[Swap]
+) -> Optional[Swap]:
+    for swap in swaps:
+        if swap.pool_address == address and swap.token_in_address == token_address:
+            return swap
 
-    if max_route_length is not None and max_route_length < 2:
-        return None
-
-    for end_swap in end_swaps:
-        if _swap_outs_match_swap_ins(start_swap, end_swap):
-            return [start_swap, end_swap]
-
-    if max_route_length is not None and max_route_length == 2:
-        return None
-
-    other_swaps = [
-        swap for swap in all_swaps if (swap is not start_swap and swap not in end_swaps)
-    ]
-
-    if len(other_swaps) == 0:
-        return None
-
-    shortest_remaining_route = None
-    max_remaining_route_length = (
-        None if max_route_length is None else max_route_length - 1
-    )
-
-    for next_swap in other_swaps:
-        if _swap_outs_match_swap_ins(start_swap, next_swap):
-            shortest_from_next = _get_shortest_route(
-                next_swap,
-                end_swaps,
-                other_swaps,
-                max_route_length=max_remaining_route_length,
-            )
-
-            if shortest_from_next is not None and (
-                shortest_remaining_route is None
-                or len(shortest_from_next) < len(shortest_remaining_route)
-            ):
-                shortest_remaining_route = shortest_from_next
-                max_remaining_route_length = len(shortest_from_next) - 1
-
-    if shortest_remaining_route is None:
-        return None
-    else:
-        return [start_swap] + shortest_remaining_route
-
-
-def _get_all_start_end_swaps(swaps: List[Swap]) -> List[Tuple[Swap, List[Swap]]]:
-    """
-    Gets the set of all possible openings and corresponding closing swaps for an arbitrage via
-    - swap[start].token_in == swap[end].token_out
-    - swap[start].from_address == swap[end].to_address
-    - not swap[start].from_address in all_pool_addresses
-    - not swap[end].to_address in all_pool_addresses
-    """
-    pool_addrs = [swap.contract_address for swap in swaps]
-    valid_start_ends: List[Tuple[Swap, List[Swap]]] = []
-
-    for index, potential_start_swap in enumerate(swaps):
-        ends_for_start: List[Swap] = []
-        remaining_swaps = swaps[:index] + swaps[index + 1 :]
-
-        for potential_end_swap in remaining_swaps:
-            if (
-                potential_start_swap.token_in_address
-                == potential_end_swap.token_out_address
-                and potential_start_swap.contract_address
-                != potential_end_swap.contract_address
-                and potential_start_swap.from_address == potential_end_swap.to_address
-                and not potential_start_swap.from_address in pool_addrs
-            ):
-
-                ends_for_start.append(potential_end_swap)
-
-        if len(ends_for_start) > 0:
-            valid_start_ends.append((potential_start_swap, ends_for_start))
-
-    return valid_start_ends
-
-
-def _swap_outs_match_swap_ins(swap_out, swap_in) -> bool:
-    return (
-        swap_out.token_out_address == swap_in.token_in_address
-        and (
-            swap_out.contract_address == swap_in.from_address
-            or swap_out.to_address == swap_in.contract_address
-            or swap_out.to_address == swap_in.from_address
-        )
-        and equal_within_percent(
-            swap_out.token_out_amount,
-            swap_in.token_in_amount,
-            MAX_TOKEN_AMOUNT_PERCENT_DIFFERENCE,
-        )
-    )
+    return None

mev_inspect/block.py

@@ -1,208 +1,62 @@
-import asyncio
-import logging
-from typing import List, Optional
+from pathlib import Path
+from typing import List
 
-from sqlalchemy import orm
 from web3 import Web3
 
 from mev_inspect.fees import fetch_base_fee_per_gas
-from mev_inspect.schemas.blocks import Block
+from mev_inspect.schemas import Block, Trace, TraceType
 from mev_inspect.schemas.receipts import Receipt
-from mev_inspect.schemas.traces import Trace, TraceType
-from mev_inspect.utils import hex_to_int
-
-logger = logging.getLogger(__name__)
 
 
-async def get_latest_block_number(base_provider) -> int:
-    latest_block = await base_provider.make_request(
-        "eth_getBlockByNumber",
-        ["latest", False],
-    )
-
-    return hex_to_int(latest_block["result"]["number"])
+cache_directory = "./cache"
 
 
-async def create_from_block_number(
-    w3: Web3,
-    block_number: int,
-    trace_db_session: Optional[orm.Session],
+def get_latest_block_number(w3: Web3) -> int:
+    return int(w3.eth.get_block("latest")["number"])
+
+
+def create_from_block_number(
+    base_provider, w3: Web3, block_number: int, should_cache: bool
 ) -> Block:
-    block_timestamp, receipts, traces, base_fee_per_gas = await asyncio.gather(
-        _find_or_fetch_block_timestamp(w3, block_number, trace_db_session),
-        _find_or_fetch_block_receipts(w3, block_number, trace_db_session),
-        _find_or_fetch_block_traces(w3, block_number, trace_db_session),
-        _find_or_fetch_base_fee_per_gas(w3, block_number, trace_db_session),
-    )
-    miner_address = await _find_or_fetch_miner_address(w3, block_number, traces)
+    if not should_cache:
+        return fetch_block(w3, base_provider, block_number)
+
+    cache_path = _get_cache_path(block_number)
+
+    if cache_path.is_file():
+        print(f"Cache for block {block_number} exists, " "loading data from cache")
+
+        return Block.parse_file(cache_path)
+    else:
+        print(f"Cache for block {block_number} did not exist, getting data")
+
+        block = fetch_block(w3, base_provider, block_number)
+
+        cache_block(cache_path, block)
+
+        return block
+
+
+def fetch_block(w3, base_provider, block_number: int) -> Block:
+    block_json = w3.eth.get_block(block_number)
+    receipts_json = base_provider.make_request("eth_getBlockReceipts", [block_number])
+    traces_json = w3.parity.trace_block(block_number)
+
+    receipts: List[Receipt] = [
+        Receipt(**receipt) for receipt in receipts_json["result"]
+    ]
+    traces = [Trace(**trace_json) for trace_json in traces_json]
+    base_fee_per_gas = fetch_base_fee_per_gas(w3, block_number)
 
     return Block(
         block_number=block_number,
-        block_timestamp=block_timestamp,
-        miner=miner_address,
+        miner=block_json["miner"],
         base_fee_per_gas=base_fee_per_gas,
         traces=traces,
         receipts=receipts,
     )
 
 
-async def _find_or_fetch_block_timestamp(
-    w3,
-    block_number: int,
-    trace_db_session: Optional[orm.Session],
-) -> int:
-    if trace_db_session is not None:
-        existing_block_timestamp = _find_block_timestamp(trace_db_session, block_number)
-        if existing_block_timestamp is not None:
-            return existing_block_timestamp
-
-    return await _fetch_block_timestamp(w3, block_number)
-
-
-async def _find_or_fetch_block_receipts(
-    w3,
-    block_number: int,
-    trace_db_session: Optional[orm.Session],
-) -> List[Receipt]:
-    if trace_db_session is not None:
-        existing_block_receipts = _find_block_receipts(trace_db_session, block_number)
-        if existing_block_receipts is not None:
-            return existing_block_receipts
-
-    return await _fetch_block_receipts(w3, block_number)
-
-
-async def _find_or_fetch_block_traces(
-    w3,
-    block_number: int,
-    trace_db_session: Optional[orm.Session],
-) -> List[Trace]:
-    if trace_db_session is not None:
-        existing_block_traces = _find_block_traces(trace_db_session, block_number)
-        if existing_block_traces is not None:
-            return existing_block_traces
-
-    return await _fetch_block_traces(w3, block_number)
-
-
-async def _find_or_fetch_base_fee_per_gas(
-    w3,
-    block_number: int,
-    trace_db_session: Optional[orm.Session],
-) -> int:
-    if trace_db_session is not None:
-        existing_base_fee_per_gas = _find_base_fee_per_gas(
-            trace_db_session, block_number
-        )
-        if existing_base_fee_per_gas is not None:
-            return existing_base_fee_per_gas
-
-    return await fetch_base_fee_per_gas(w3, block_number)
-
-
-async def _fetch_block_timestamp(w3, block_number: int) -> int:
-    block_json = await w3.eth.get_block(block_number)
-    return block_json["timestamp"]
-
-
-async def _fetch_block_receipts(w3, block_number: int) -> List[Receipt]:
-    receipts_json = await w3.eth.get_block_receipts(block_number)
-    return [Receipt(**receipt) for receipt in receipts_json]
-
-
-async def _fetch_block_traces(w3, block_number: int) -> List[Trace]:
-    traces_json = await w3.eth.trace_block(block_number)
-    return [Trace(**trace_json) for trace_json in traces_json]
-
-
-def _find_block_timestamp(
-    trace_db_session: orm.Session,
-    block_number: int,
-) -> Optional[int]:
-    result = trace_db_session.execute(
-        "SELECT block_timestamp FROM block_timestamps WHERE block_number = :block_number",
-        params={"block_number": block_number},
-    ).one_or_none()
-
-    if result is None:
-        return None
-    else:
-        (block_timestamp,) = result
-        return block_timestamp
-
-
-def _find_block_traces(
-    trace_db_session: orm.Session,
-    block_number: int,
-) -> Optional[List[Trace]]:
-    result = trace_db_session.execute(
-        "SELECT raw_traces FROM block_traces WHERE block_number = :block_number",
-        params={"block_number": block_number},
-    ).one_or_none()
-
-    if result is None:
-        return None
-    else:
-        (traces_json,) = result
-        return [Trace(**trace_json) for trace_json in traces_json]
-
-
-def _find_block_receipts(
-    trace_db_session: orm.Session,
-    block_number: int,
-) -> Optional[List[Receipt]]:
-    result = trace_db_session.execute(
-        "SELECT raw_receipts FROM block_receipts WHERE block_number = :block_number",
-        params={"block_number": block_number},
-    ).one_or_none()
-
-    if result is None:
-        return None
-    else:
-        (receipts_json,) = result
-        return [Receipt(**receipt) for receipt in receipts_json]
-
-
-def _find_base_fee_per_gas(
-    trace_db_session: orm.Session,
-    block_number: int,
-) -> Optional[int]:
-    result = trace_db_session.execute(
-        "SELECT base_fee_in_wei FROM base_fee WHERE block_number = :block_number",
-        params={"block_number": block_number},
-    ).one_or_none()
-
-    if result is None:
-        return None
-    else:
-        (base_fee,) = result
-        return base_fee
-
-
-async def _find_or_fetch_miner_address(
-    w3,
-    block_number: int,
-    traces: List[Trace],
-) -> Optional[str]:
-    # eth1 blocks
-    miner_address = _get_miner_address_from_traces(traces)
-    if miner_address is not None:
-        return miner_address
-    return await _fetch_miner_eth2(w3, block_number)
-
-
-async def _fetch_miner_eth2(w3, block_number: int) -> Optional[str]:
-    block_json = await w3.eth.get_block(block_number)
-    return block_json["miner"]
-
-
-def _get_miner_address_from_traces(traces: List[Trace]) -> Optional[str]:
-    for trace in traces:
-        if trace.type == TraceType.reward:
-            return trace.action["author"]
-    return None
-
-
 def get_transaction_hashes(calls: List[Trace]) -> List[str]:
     result = []
 
@@ -215,3 +69,17 @@ def get_transaction_hashes(calls: List[Trace]) -> List[str]:
                 result.append(call.transaction_hash)
 
     return result
+
+
+def cache_block(cache_path: Path, block: Block):
+    write_mode = "w" if cache_path.is_file() else "x"
+
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(cache_path, mode=write_mode) as cache_file:
+        cache_file.write(block.json())
+
+
+def _get_cache_path(block_number: int) -> Path:
+    cache_directory_path = Path(cache_directory)
+    return cache_directory_path / f"{block_number}-new.json"

mev_inspect/classifiers/helpers.py

@@ -1,208 +0,0 @@
-from typing import List, Optional, Sequence
-
-from mev_inspect.schemas.nft_trades import NftTrade
-from mev_inspect.schemas.prices import ETH_TOKEN_ADDRESS
-from mev_inspect.schemas.swaps import Swap
-from mev_inspect.schemas.traces import ClassifiedTrace, DecodedCallTrace
-from mev_inspect.schemas.transfers import Transfer
-
-
-def create_nft_trade_from_transfers(
-    trace: DecodedCallTrace,
-    child_transfers: List[Transfer],
-    collection_address: str,
-    seller_address: str,
-    buyer_address: str,
-    exchange_wallet_address: str,
-) -> Optional[NftTrade]:
-    transfers_to_buyer = _filter_transfers(child_transfers, to_address=buyer_address)
-    transfers_to_seller = _filter_transfers(child_transfers, to_address=seller_address)
-
-    if len(transfers_to_buyer) != 1 or len(transfers_to_seller) != 1:
-        return None
-
-    if transfers_to_buyer[0].token_address != collection_address:
-        return None
-
-    payment_token_address = transfers_to_seller[0].token_address
-    payment_amount = transfers_to_seller[0].amount
-    token_id = transfers_to_buyer[0].amount
-
-    transfers_from_seller_to_exchange = _filter_transfers(
-        child_transfers,
-        from_address=seller_address,
-        to_address=exchange_wallet_address,
-    )
-    transfers_from_buyer_to_exchange = _filter_transfers(
-        child_transfers,
-        from_address=buyer_address,
-        to_address=exchange_wallet_address,
-    )
-    for fee in [
-        *transfers_from_seller_to_exchange,
-        *transfers_from_buyer_to_exchange,
-    ]:
-        # Assumes that exchange fees are paid with the same token as the sale
-        payment_amount -= fee.amount
-
-    return NftTrade(
-        abi_name=trace.abi_name,
-        transaction_hash=trace.transaction_hash,
-        transaction_position=trace.transaction_position,
-        block_number=trace.block_number,
-        trace_address=trace.trace_address,
-        protocol=trace.protocol,
-        error=trace.error,
-        seller_address=seller_address,
-        buyer_address=buyer_address,
-        payment_token_address=payment_token_address,
-        payment_amount=payment_amount,
-        collection_address=collection_address,
-        token_id=token_id,
-    )
-
-
-def create_swap_from_pool_transfers(
-    trace: DecodedCallTrace,
-    recipient_address: str,
-    prior_transfers: List[Transfer],
-    child_transfers: List[Transfer],
-) -> Optional[Swap]:
-    pool_address = trace.to_address
-
-    transfers_to_pool = []
-
-    if trace.value is not None and trace.value > 0:
-        transfers_to_pool = [_build_eth_transfer(trace)]
-
-    if len(transfers_to_pool) == 0:
-        transfers_to_pool = _filter_transfers(prior_transfers, to_address=pool_address)
-
-    if len(transfers_to_pool) == 0:
-        transfers_to_pool = _filter_transfers(child_transfers, to_address=pool_address)
-
-    if len(transfers_to_pool) == 0:
-        return None
-
-    transfers_from_pool_to_recipient = _filter_transfers(
-        child_transfers, to_address=recipient_address, from_address=pool_address
-    )
-
-    if len(transfers_from_pool_to_recipient) != 1:
-        return None
-
-    transfer_in = transfers_to_pool[-1]
-    transfer_out = transfers_from_pool_to_recipient[0]
-
-    if transfer_in.token_address == transfer_out.token_address:
-        return None
-
-    return Swap(
-        abi_name=trace.abi_name,
-        transaction_hash=trace.transaction_hash,
-        transaction_position=trace.transaction_position,
-        block_number=trace.block_number,
-        trace_address=trace.trace_address,
-        contract_address=pool_address,
-        protocol=trace.protocol,
-        from_address=transfer_in.from_address,
-        to_address=transfer_out.to_address,
-        token_in_address=transfer_in.token_address,
-        token_in_amount=transfer_in.amount,
-        token_out_address=transfer_out.token_address,
-        token_out_amount=transfer_out.amount,
-        error=trace.error,
-    )
-
-
-def create_swap_from_recipient_transfers(
-    trace: DecodedCallTrace,
-    pool_address: str,
-    recipient_address: str,
-    prior_transfers: List[Transfer],
-    child_transfers: List[Transfer],
-) -> Optional[Swap]:
-    transfers_from_recipient = _filter_transfers(
-        [*prior_transfers, *child_transfers], from_address=recipient_address
-    )
-    transfers_to_recipient = _filter_transfers(
-        child_transfers, to_address=recipient_address
-    )
-
-    if len(transfers_from_recipient) != 1 or len(transfers_to_recipient) != 1:
-        return None
-
-    transfer_in = transfers_from_recipient[0]
-    transfer_out = transfers_to_recipient[0]
-
-    return Swap(
-        abi_name=trace.abi_name,
-        transaction_hash=trace.transaction_hash,
-        transaction_position=trace.transaction_position,
-        block_number=trace.block_number,
-        trace_address=trace.trace_address,
-        contract_address=pool_address,
-        protocol=trace.protocol,
-        from_address=transfer_in.from_address,
-        to_address=transfer_out.to_address,
-        token_in_address=transfer_in.token_address,
-        token_in_amount=transfer_in.amount,
-        token_out_address=transfer_out.token_address,
-        token_out_amount=transfer_out.amount,
-        error=trace.error,
-    )
-
-
-def _build_eth_transfer(trace: ClassifiedTrace) -> Transfer:
-    return Transfer(
-        block_number=trace.block_number,
-        transaction_hash=trace.transaction_hash,
-        trace_address=trace.trace_address,
-        amount=trace.value,
-        to_address=trace.to_address,
-        from_address=trace.from_address,
-        token_address=ETH_TOKEN_ADDRESS,
-    )
-
-
-def _filter_transfers(
-    transfers: Sequence[Transfer],
-    to_address: Optional[str] = None,
-    from_address: Optional[str] = None,
-) -> List[Transfer]:
-    filtered_transfers = []
-
-    for transfer in transfers:
-        if to_address is not None and transfer.to_address != to_address:
-            continue
-
-        if from_address is not None and transfer.from_address != from_address:
-            continue
-
-        filtered_transfers.append(transfer)
-
-    return filtered_transfers
-
-
-def get_received_transfer(
-    liquidator: str, child_transfers: List[Transfer]
-) -> Optional[Transfer]:
-    """Get transfer from AAVE to liquidator"""
-
-    for transfer in child_transfers:
-        if transfer.to_address == liquidator:
-            return transfer
-
-    return None
-
-
-def get_debt_transfer(
-    liquidator: str, child_transfers: List[Transfer]
-) -> Optional[Transfer]:
-    """Get transfer from liquidator to AAVE"""
-
-    for transfer in child_transfers:
-        if transfer.from_address == liquidator:
-            return transfer
-
-    return None

mev_inspect/classifiers/specs/__init__.py

@@ -1,21 +1,11 @@
-from typing import Dict, Optional, Tuple, Type
-
-from mev_inspect.schemas.classifiers import Classifier, ClassifierSpec
-from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
-
 from .aave import AAVE_CLASSIFIER_SPECS
-from .balancer import BALANCER_CLASSIFIER_SPECS
-from .bancor import BANCOR_CLASSIFIER_SPECS
-from .compound import COMPOUND_CLASSIFIER_SPECS
-from .cream import CREAM_CLASSIFIER_SPECS
-from .cryptopunks import CRYPTOPUNKS_CLASSIFIER_SPECS
 from .curve import CURVE_CLASSIFIER_SPECS
 from .erc20 import ERC20_CLASSIFIER_SPECS
-from .opensea import OPENSEA_CLASSIFIER_SPECS
 from .uniswap import UNISWAP_CLASSIFIER_SPECS
 from .weth import WETH_CLASSIFIER_SPECS
 from .zero_ex import ZEROX_CLASSIFIER_SPECS
 
+
 ALL_CLASSIFIER_SPECS = (
     ERC20_CLASSIFIER_SPECS
     + WETH_CLASSIFIER_SPECS
@@ -23,26 +13,4 @@ ALL_CLASSIFIER_SPECS = (
     + UNISWAP_CLASSIFIER_SPECS
     + AAVE_CLASSIFIER_SPECS
     + ZEROX_CLASSIFIER_SPECS
-    + BALANCER_CLASSIFIER_SPECS
-    + COMPOUND_CLASSIFIER_SPECS
-    + CREAM_CLASSIFIER_SPECS
-    + CRYPTOPUNKS_CLASSIFIER_SPECS
-    + OPENSEA_CLASSIFIER_SPECS
-    + BANCOR_CLASSIFIER_SPECS
 )
-
-_SPECS_BY_ABI_NAME_AND_PROTOCOL: Dict[
-    Tuple[str, Optional[Protocol]], ClassifierSpec
-] = {(spec.abi_name, spec.protocol): spec for spec in ALL_CLASSIFIER_SPECS}
-
-
-def get_classifier(
-    trace: DecodedCallTrace,
-) -> Optional[Type[Classifier]]:
-    abi_name_and_protocol = (trace.abi_name, trace.protocol)
-    spec = _SPECS_BY_ABI_NAME_AND_PROTOCOL.get(abi_name_and_protocol)
-
-    if spec is not None:
-        return spec.classifiers.get(trace.function_signature)
-
-    return None

mev_inspect/classifiers/specs/aave.py

@@ -1,93 +1,15 @@
-from typing import List, Optional
-
-from mev_inspect.classifiers.helpers import get_debt_transfer, get_received_transfer
-from mev_inspect.schemas.classifiers import (
-    ClassifiedTrace,
+from mev_inspect.schemas.classified_traces import (
+    Classification,
     ClassifierSpec,
-    DecodedCallTrace,
-    LiquidationClassifier,
-    TransferClassifier,
+    Protocol,
 )
-from mev_inspect.schemas.liquidations import Liquidation
-from mev_inspect.schemas.traces import Protocol
-from mev_inspect.schemas.transfers import Transfer
-
-
-class AaveLiquidationClassifier(LiquidationClassifier):
-    @staticmethod
-    def parse_liquidation(
-        liquidation_trace: DecodedCallTrace,
-        child_transfers: List[Transfer],
-        child_traces: List[ClassifiedTrace],
-    ) -> Optional[Liquidation]:
-
-        liquidator = liquidation_trace.from_address
-        liquidated = liquidation_trace.inputs["_user"]
-
-        debt_token_address = liquidation_trace.inputs["_reserve"]
-        received_token_address = liquidation_trace.inputs["_collateral"]
-
-        debt_purchase_amount = None
-        received_amount = None
-
-        debt_transfer = get_debt_transfer(liquidator, child_transfers)
-
-        received_transfer = get_received_transfer(liquidator, child_transfers)
-
-        if debt_transfer is not None and received_transfer is not None:
-
-            debt_token_address = debt_transfer.token_address
-            debt_purchase_amount = debt_transfer.amount
-
-            received_token_address = received_transfer.token_address
-            received_amount = received_transfer.amount
-
-            return Liquidation(
-                liquidated_user=liquidated,
-                debt_token_address=debt_token_address,
-                liquidator_user=liquidator,
-                debt_purchase_amount=debt_purchase_amount,
-                protocol=Protocol.aave,
-                received_amount=received_amount,
-                received_token_address=received_token_address,
-                transaction_hash=liquidation_trace.transaction_hash,
-                trace_address=liquidation_trace.trace_address,
-                block_number=liquidation_trace.block_number,
-                error=liquidation_trace.error,
-            )
-
-        else:
-            return None
-
-
-class AaveTransferClassifier(TransferClassifier):
-    @staticmethod
-    def get_transfer(trace: DecodedCallTrace) -> Transfer:
-        return Transfer(
-            block_number=trace.block_number,
-            transaction_hash=trace.transaction_hash,
-            trace_address=trace.trace_address,
-            amount=trace.inputs["value"],
-            to_address=trace.inputs["to"],
-            from_address=trace.inputs["from"],
-            token_address=trace.to_address,
-        )
-
 
 AAVE_SPEC = ClassifierSpec(
     abi_name="AaveLendingPool",
     protocol=Protocol.aave,
-    classifiers={
-        "liquidationCall(address,address,address,uint256,bool)": AaveLiquidationClassifier,
+    classifications={
+        "liquidationCall(address,address,address,uint256,bool)": Classification.liquidate,
     },
 )
 
-ATOKENS_SPEC = ClassifierSpec(
-    abi_name="aTokens",
-    protocol=Protocol.aave,
-    classifiers={
-        "transferOnLiquidation(address,address,uint256)": AaveTransferClassifier,
-    },
-)
-
-AAVE_CLASSIFIER_SPECS: List[ClassifierSpec] = [AAVE_SPEC, ATOKENS_SPEC]
+AAVE_CLASSIFIER_SPECS = [AAVE_SPEC]

mev_inspect/classifiers/specs/balancer.py

@@ -1,41 +0,0 @@
-from typing import List, Optional
-
-from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
-from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier
-from mev_inspect.schemas.swaps import Swap
-from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
-from mev_inspect.schemas.transfers import Transfer
-
-BALANCER_V1_POOL_ABI_NAME = "BPool"
-
-
-class BalancerSwapClassifier(SwapClassifier):
-    @staticmethod
-    def parse_swap(
-        trace: DecodedCallTrace,
-        prior_transfers: List[Transfer],
-        child_transfers: List[Transfer],
-    ) -> Optional[Swap]:
-
-        recipient_address = trace.from_address
-
-        swap = create_swap_from_pool_transfers(
-            trace, recipient_address, prior_transfers, child_transfers
-        )
-        return swap
-
-
-BALANCER_V1_SPECS = [
-    ClassifierSpec(
-        abi_name=BALANCER_V1_POOL_ABI_NAME,
-        protocol=Protocol.balancer_v1,
-        classifiers={
-            "swapExactAmountIn(address,uint256,address,uint256,uint256)": BalancerSwapClassifier,
-            "swapExactAmountOut(address,uint256,address,uint256,uint256)": BalancerSwapClassifier,
-        },
-    ),
-]
-
-BALANCER_CLASSIFIER_SPECS = [
-    *BALANCER_V1_SPECS,
-]

mev_inspect/classifiers/specs/bancor.py

@@ -1,41 +0,0 @@
-from typing import List, Optional
-
-from mev_inspect.classifiers.helpers import create_swap_from_recipient_transfers
-from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier
-from mev_inspect.schemas.swaps import Swap
-from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
-from mev_inspect.schemas.transfers import Transfer
-
-BANCOR_NETWORK_ABI_NAME = "BancorNetwork"
-BANCOR_NETWORK_CONTRACT_ADDRESS = "0x2F9EC37d6CcFFf1caB21733BdaDEdE11c823cCB0"
-
-
-class BancorSwapClassifier(SwapClassifier):
-    @staticmethod
-    def parse_swap(
-        trace: DecodedCallTrace,
-        prior_transfers: List[Transfer],
-        child_transfers: List[Transfer],
-    ) -> Optional[Swap]:
-        recipient_address = trace.from_address
-
-        swap = create_swap_from_recipient_transfers(
-            trace,
-            BANCOR_NETWORK_CONTRACT_ADDRESS,
-            recipient_address,
-            prior_transfers,
-            child_transfers,
-        )
-        return swap
-
-
-BANCOR_NETWORK_SPEC = ClassifierSpec(
-    abi_name=BANCOR_NETWORK_ABI_NAME,
-    protocol=Protocol.bancor,
-    classifiers={
-        "convertByPath(address[],uint256,uint256,address,address,uint256)": BancorSwapClassifier,
-    },
-    valid_contract_addresses=[BANCOR_NETWORK_CONTRACT_ADDRESS],
-)
-
-BANCOR_CLASSIFIER_SPECS = [BANCOR_NETWORK_SPEC]

mev_inspect/classifiers/specs/compound.py

@@ -1,127 +0,0 @@
-from typing import List, Optional
-
-from mev_inspect.classifiers.helpers import get_debt_transfer, get_received_transfer
-from mev_inspect.schemas.classifiers import (
-    Classification,
-    ClassifiedTrace,
-    ClassifierSpec,
-    DecodedCallTrace,
-    LiquidationClassifier,
-    SeizeClassifier,
-)
-from mev_inspect.schemas.liquidations import Liquidation
-from mev_inspect.schemas.prices import CETH_TOKEN_ADDRESS, ETH_TOKEN_ADDRESS
-from mev_inspect.schemas.traces import Protocol
-from mev_inspect.schemas.transfers import Transfer
-
-
-class CompoundLiquidationClassifier(LiquidationClassifier):
-    @staticmethod
-    def parse_liquidation(
-        liquidation_trace: DecodedCallTrace,
-        child_transfers: List[Transfer],
-        child_traces: List[ClassifiedTrace],
-    ) -> Optional[Liquidation]:
-
-        liquidator = liquidation_trace.from_address
-        liquidated = liquidation_trace.inputs["borrower"]
-
-        debt_token_address = liquidation_trace.to_address
-        received_token_address = liquidation_trace.inputs["cTokenCollateral"]
-
-        debt_purchase_amount = None
-        received_amount = None
-
-        debt_purchase_amount, debt_token_address = (
-            (liquidation_trace.value, ETH_TOKEN_ADDRESS)
-            if debt_token_address == CETH_TOKEN_ADDRESS and liquidation_trace.value != 0
-            else (liquidation_trace.inputs["repayAmount"], CETH_TOKEN_ADDRESS)
-        )
-
-        debt_transfer = get_debt_transfer(liquidator, child_transfers)
-
-        received_transfer = get_received_transfer(liquidator, child_transfers)
-
-        seize_trace = _get_seize_call(child_traces)
-
-        if debt_transfer is not None:
-            debt_token_address = debt_transfer.token_address
-            debt_purchase_amount = debt_transfer.amount
-
-        if received_transfer is not None:
-            received_token_address = received_transfer.token_address
-            received_amount = received_transfer.amount
-
-        elif seize_trace is not None and seize_trace.inputs is not None:
-            received_amount = seize_trace.inputs["seizeTokens"]
-
-        if received_amount is None:
-            return None
-
-        return Liquidation(
-            liquidated_user=liquidated,
-            debt_token_address=debt_token_address,
-            liquidator_user=liquidator,
-            debt_purchase_amount=debt_purchase_amount,
-            protocol=liquidation_trace.protocol,
-            received_amount=received_amount,
-            received_token_address=received_token_address,
-            transaction_hash=liquidation_trace.transaction_hash,
-            trace_address=liquidation_trace.trace_address,
-            block_number=liquidation_trace.block_number,
-            error=liquidation_trace.error,
-        )
-
-        return None
-
-
-COMPOUND_V2_CETH_SPEC = ClassifierSpec(
-    abi_name="CEther",
-    protocol=Protocol.compound_v2,
-    valid_contract_addresses=["0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5"],
-    classifiers={
-        "liquidateBorrow(address,address)": CompoundLiquidationClassifier,
-        "seize(address,address,uint256)": SeizeClassifier,
-    },
-)
-
-COMPOUND_V2_CTOKEN_SPEC = ClassifierSpec(
-    abi_name="CToken",
-    protocol=Protocol.compound_v2,
-    valid_contract_addresses=[
-        "0x6c8c6b02e7b2be14d4fa6022dfd6d75921d90e4e",
-        "0x5d3a536e4d6dbd6114cc1ead35777bab948e3643",
-        "0x158079ee67fce2f58472a96584a73c7ab9ac95c1",
-        "0x39aa39c021dfbae8fac545936693ac917d5e7563",
-        "0xf650c3d88d12db855b8bf7d11be6c55a4e07dcc9",
-        "0xc11b1268c1a384e55c48c2391d8d480264a3a7f4",
-        "0xb3319f5d18bc0d84dd1b4825dcde5d5f7266d407",
-        "0xf5dce57282a584d2746faf1593d3121fcac444dc",
-        "0x35a18000230da775cac24873d00ff85bccded550",
-        "0x70e36f6bf80a52b3b46b3af8e106cc0ed743e8e4",
-        "0xccf4429db6322d5c611ee964527d42e5d685dd6a",
-        "0x12392f67bdf24fae0af363c24ac620a2f67dad86",
-        "0xface851a4921ce59e912d19329929ce6da6eb0c7",
-        "0x95b4ef2869ebd94beb4eee400a99824bf5dc325b",
-        "0x4b0181102a0112a2ef11abee5563bb4a3176c9d7",
-        "0xe65cdb6479bac1e22340e4e755fae7e509ecd06c",
-        "0x80a2ae356fc9ef4305676f7a3e2ed04e12c33946",
-    ],
-    classifiers={
-        "liquidateBorrow(address,uint256,address)": CompoundLiquidationClassifier,
-        "seize(address,address,uint256)": SeizeClassifier,
-    },
-)
-
-COMPOUND_CLASSIFIER_SPECS: List[ClassifierSpec] = [
-    COMPOUND_V2_CETH_SPEC,
-    COMPOUND_V2_CTOKEN_SPEC,
-]
-
-
-def _get_seize_call(traces: List[ClassifiedTrace]) -> Optional[ClassifiedTrace]:
-    """Find the call to `seize` in the child traces (successful liquidation)"""
-    for trace in traces:
-        if trace.classification == Classification.seize:
-            return trace
-    return None

mev_inspect/classifiers/specs/cream.py

@@ -1,204 +0,0 @@
-from typing import List, Optional
-
-from mev_inspect.classifiers.helpers import get_debt_transfer, get_received_transfer
-from mev_inspect.schemas.classifiers import (
-    Classification,
-    ClassifiedTrace,
-    ClassifierSpec,
-    DecodedCallTrace,
-    LiquidationClassifier,
-    SeizeClassifier,
-)
-from mev_inspect.schemas.liquidations import Liquidation
-from mev_inspect.schemas.prices import ETH_TOKEN_ADDRESS
-from mev_inspect.schemas.traces import Protocol
-from mev_inspect.schemas.transfers import Transfer
-
-CRETH_TOKEN_ADDRESS = "0xd06527d5e56a3495252a528c4987003b712860ee"
-
-
-class CreamLiquidationClassifier(LiquidationClassifier):
-    @staticmethod
-    def parse_liquidation(
-        liquidation_trace: DecodedCallTrace,
-        child_transfers: List[Transfer],
-        child_traces: List[ClassifiedTrace],
-    ) -> Optional[Liquidation]:
-
-        liquidator = liquidation_trace.from_address
-        liquidated = liquidation_trace.inputs["borrower"]
-
-        debt_token_address = liquidation_trace.to_address
-        received_token_address = liquidation_trace.inputs["cTokenCollateral"]
-
-        debt_purchase_amount = None
-        received_amount = None
-
-        debt_purchase_amount, debt_token_address = (
-            (liquidation_trace.value, ETH_TOKEN_ADDRESS)
-            if debt_token_address == CRETH_TOKEN_ADDRESS
-            and liquidation_trace.value != 0
-            else (liquidation_trace.inputs["repayAmount"], CRETH_TOKEN_ADDRESS)
-        )
-
-        debt_transfer = get_debt_transfer(liquidator, child_transfers)
-
-        received_transfer = get_received_transfer(liquidator, child_transfers)
-
-        seize_trace = _get_seize_call(child_traces)
-
-        if debt_transfer is not None:
-            debt_token_address = debt_transfer.token_address
-            debt_purchase_amount = debt_transfer.amount
-
-        if received_transfer is not None:
-            received_token_address = received_transfer.token_address
-            received_amount = received_transfer.amount
-
-        elif seize_trace is not None and seize_trace.inputs is not None:
-            received_amount = seize_trace.inputs["seizeTokens"]
-
-        if received_amount is None:
-            return None
-
-        return Liquidation(
-            liquidated_user=liquidated,
-            debt_token_address=debt_token_address,
-            liquidator_user=liquidator,
-            debt_purchase_amount=debt_purchase_amount,
-            protocol=liquidation_trace.protocol,
-            received_amount=received_amount,
-            received_token_address=received_token_address,
-            transaction_hash=liquidation_trace.transaction_hash,
-            trace_address=liquidation_trace.trace_address,
-            block_number=liquidation_trace.block_number,
-            error=liquidation_trace.error,
-        )
-
-        return None
-
-
-CREAM_CRETH_SPEC = ClassifierSpec(
-    abi_name="CEther",
-    protocol=Protocol.cream,
-    valid_contract_addresses=["0xD06527D5e56A3495252A528C4987003b712860eE"],
-    classifiers={
-        "liquidateBorrow(address,address)": CreamLiquidationClassifier,
-        "seize(address,address,uint256)": SeizeClassifier,
-    },
-)
-
-CREAM_CTOKEN_SPEC = ClassifierSpec(
-    abi_name="CToken",
-    protocol=Protocol.cream,
-    valid_contract_addresses=[
-        "0xd06527d5e56a3495252a528c4987003b712860ee",
-        "0x51f48b638f82e8765f7a26373a2cb4ccb10c07af",
-        "0x44fbebd2f576670a6c33f6fc0b00aa8c5753b322",
-        "0xcbae0a83f4f9926997c8339545fb8ee32edc6b76",
-        "0xce4fe9b4b8ff61949dcfeb7e03bc9faca59d2eb3",
-        "0x19d1666f543d42ef17f66e376944a22aea1a8e46",
-        "0x9baf8a5236d44ac410c0186fe39178d5aad0bb87",
-        "0x797aab1ce7c01eb727ab980762ba88e7133d2157",
-        "0x892b14321a4fcba80669ae30bd0cd99a7ecf6ac0",
-        "0x697256caa3ccafd62bb6d3aa1c7c5671786a5fd9",
-        "0x8b86e0598616a8d4f1fdae8b59e55fb5bc33d0d6",
-        "0xc7fd8dcee4697ceef5a2fd4608a7bd6a94c77480",
-        "0x17107f40d70f4470d20cb3f138a052cae8ebd4be",
-        "0x1ff8cdb51219a8838b52e9cac09b71e591bc998e",
-        "0x3623387773010d9214b10c551d6e7fc375d31f58",
-        "0x4ee15f44c6f0d8d1136c83efd2e8e4ac768954c6",
-        "0x338286c0bc081891a4bda39c7667ae150bf5d206",
-        "0x10fdbd1e48ee2fd9336a482d746138ae19e649db",
-        "0x01da76dea59703578040012357b81ffe62015c2d",
-        "0xef58b2d5a1b8d3cde67b8ab054dc5c831e9bc025",
-        "0xe89a6d0509faf730bd707bf868d9a2a744a363c7",
-        "0xeff039c3c1d668f408d09dd7b63008622a77532c",
-        "0x22b243b96495c547598d9042b6f94b01c22b2e9e",
-        "0x8b3ff1ed4f36c2c2be675afb13cc3aa5d73685a5",
-        "0x2a537fa9ffaea8c1a41d3c2b68a9cb791529366d",
-        "0x7ea9c63e216d5565c3940a2b3d150e59c2907db3",
-        "0x3225e3c669b39c7c8b3e204a8614bb218c5e31bc",
-        "0xf55bbe0255f7f4e70f63837ff72a577fbddbe924",
-        "0x903560b1cce601794c584f58898da8a8b789fc5d",
-        "0x054b7ed3f45714d3091e82aad64a1588dc4096ed",
-        "0xd5103afcd0b3fa865997ef2984c66742c51b2a8b",
-        "0xfd609a03b393f1a1cfcacedabf068cad09a924e2",
-        "0xd692ac3245bb82319a31068d6b8412796ee85d2c",
-        "0x92b767185fb3b04f881e3ac8e5b0662a027a1d9f",
-        "0x10a3da2bb0fae4d591476fd97d6636fd172923a8",
-        "0x3c6c553a95910f9fc81c98784736bd628636d296",
-        "0x21011bc93d9e515b9511a817a1ed1d6d468f49fc",
-        "0x85759961b116f1d36fd697855c57a6ae40793d9b",
-        "0x7c3297cfb4c4bbd5f44b450c0872e0ada5203112",
-        "0x7aaa323d7e398be4128c7042d197a2545f0f1fea",
-        "0x011a014d5e8eb4771e575bb1000318d509230afa",
-        "0xe6c3120f38f56deb38b69b65cc7dcaf916373963",
-        "0x4fe11bc316b6d7a345493127fbe298b95adaad85",
-        "0xcd22c4110c12ac41acefa0091c432ef44efaafa0",
-        "0x228619cca194fbe3ebeb2f835ec1ea5080dafbb2",
-        "0x73f6cba38922960b7092175c0add22ab8d0e81fc",
-        "0x38f27c03d6609a86ff7716ad03038881320be4ad",
-        "0x5ecad8a75216cea7dff978525b2d523a251eea92",
-        "0x5c291bc83d15f71fb37805878161718ea4b6aee9",
-        "0x6ba0c66c48641e220cf78177c144323b3838d375",
-        "0xd532944df6dfd5dd629e8772f03d4fc861873abf",
-        "0x197070723ce0d3810a0e47f06e935c30a480d4fc",
-        "0xc25eae724f189ba9030b2556a1533e7c8a732e14",
-        "0x25555933a8246ab67cbf907ce3d1949884e82b55",
-        "0xc68251421edda00a10815e273fa4b1191fac651b",
-        "0x65883978ada0e707c3b2be2a6825b1c4bdf76a90",
-        "0x8b950f43fcac4931d408f1fcda55c6cb6cbf3096",
-        "0x59089279987dd76fc65bf94cb40e186b96e03cb3",
-        "0x2db6c82ce72c8d7d770ba1b5f5ed0b6e075066d6",
-        "0xb092b4601850e23903a42eacbc9d8a0eec26a4d5",
-        "0x081fe64df6dc6fc70043aedf3713a3ce6f190a21",
-        "0x1d0986fb43985c88ffa9ad959cc24e6a087c7e35",
-        "0xc36080892c64821fa8e396bc1bd8678fa3b82b17",
-        "0x8379baa817c5c5ab929b03ee8e3c48e45018ae41",
-        "0x299e254a8a165bbeb76d9d69305013329eea3a3b",
-        "0xf8445c529d363ce114148662387eba5e62016e20",
-        "0x28526bb33d7230e65e735db64296413731c5402e",
-        "0x45406ba53bb84cd32a58e7098a2d4d1b11b107f6",
-        "0x6d1b9e01af17dd08d6dec08e210dfd5984ff1c20",
-        "0x1f9b4756b008106c806c7e64322d7ed3b72cb284",
-        "0xab10586c918612ba440482db77549d26b7abf8f7",
-        "0xdfff11dfe6436e42a17b86e7f419ac8292990393",
-        "0xdbb5e3081def4b6cdd8864ac2aeda4cbf778fecf",
-        "0x71cefcd324b732d4e058afacba040d908c441847",
-        "0x1a122348b73b58ea39f822a89e6ec67950c2bbd0",
-        "0x523effc8bfefc2948211a05a905f761cba5e8e9e",
-        "0x4202d97e00b9189936edf37f8d01cff88bdd81d4",
-        "0x4baa77013ccd6705ab0522853cb0e9d453579dd4",
-        "0x98e329eb5aae2125af273102f3440de19094b77c",
-        "0x8c3b7a4320ba70f8239f83770c4015b5bc4e6f91",
-        "0xe585c76573d7593abf21537b607091f76c996e73",
-        "0x81e346729723c4d15d0fb1c5679b9f2926ff13c6",
-        "0x766175eac1a99c969ddd1ebdbe7e270d508d8fff",
-        "0xd7394428536f63d5659cc869ef69d10f9e66314b",
-        "0x1241b10e7ea55b22f5b2d007e8fecdf73dcff999",
-        "0x2a867fd776b83e1bd4e13c6611afd2f6af07ea6d",
-        "0x250fb308199fe8c5220509c1bf83d21d60b7f74a",
-        "0x4112a717edd051f77d834a6703a1ef5e3d73387f",
-        "0xf04ce2e71d32d789a259428ddcd02d3c9f97fb4e",
-        "0x89e42987c39f72e2ead95a8a5bc92114323d5828",
-        "0x58da9c9fc3eb30abbcbbab5ddabb1e6e2ef3d2ef",
-    ],
-    classifiers={
-        "liquidateBorrow(address,uint256,address)": CreamLiquidationClassifier,
-        "seize(address,address,uint256)": SeizeClassifier,
-    },
-)
-
-CREAM_CLASSIFIER_SPECS: List[ClassifierSpec] = [
-    CREAM_CRETH_SPEC,
-    CREAM_CTOKEN_SPEC,
-]
-
-
-def _get_seize_call(traces: List[ClassifiedTrace]) -> Optional[ClassifiedTrace]:
-    """Find the call to `seize` in the child traces (successful liquidation)"""
-    for trace in traces:
-        if trace.classification == Classification.seize:
-            return trace
-    return None

mev_inspect/classifiers/specs/cryptopunks.py

@@ -1,27 +0,0 @@
-from mev_inspect.schemas.classifiers import Classifier, ClassifierSpec
-from mev_inspect.schemas.traces import Classification, Protocol
-
-
-class PunkBidAcceptanceClassifier(Classifier):
-    @staticmethod
-    def get_classification() -> Classification:
-        return Classification.punk_accept_bid
-
-
-class PunkBidClassifier(Classifier):
-    @staticmethod
-    def get_classification() -> Classification:
-        return Classification.punk_bid
-
-
-CRYPTO_PUNKS_SPEC = ClassifierSpec(
-    abi_name="cryptopunks",
-    protocol=Protocol.cryptopunks,
-    valid_contract_addresses=["0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB"],
-    classifiers={
-        "enterBidForPunk(uint256)": PunkBidClassifier,
-        "acceptBidForPunk(uint256,uint256)": PunkBidAcceptanceClassifier,
-    },
-)
-
-CRYPTOPUNKS_CLASSIFIER_SPECS = [CRYPTO_PUNKS_SPEC]

mev_inspect/classifiers/specs/curve.py

@@ -1,28 +1,29 @@
-from typing import List, Optional
-
-from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
-from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier
-from mev_inspect.schemas.swaps import Swap
-from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
-from mev_inspect.schemas.transfers import Transfer
-
-
-class CurveSwapClassifier(SwapClassifier):
-    @staticmethod
-    def parse_swap(
-        trace: DecodedCallTrace,
-        prior_transfers: List[Transfer],
-        child_transfers: List[Transfer],
-    ) -> Optional[Swap]:
-
-        recipient_address = trace.from_address
-
-        swap = create_swap_from_pool_transfers(
-            trace, recipient_address, prior_transfers, child_transfers
-        )
-        return swap
+from mev_inspect.schemas.classified_traces import (
+    ClassifierSpec,
+    Protocol,
+)
 
+"""
+Deployment addresses found here
+https://curve.readthedocs.io/ref-addresses.html
 
+organized into 3 groups
+1. Base Pools: 2 or more tokens implementing stable swap
+  - StableSwap<pool>
+  - Deposit<pool>
+  - CurveContract<version>
+  - CurveTokenV1/V2
+2. Meta Pools: 1 token trading with an LP from above
+  - StableSwap<pool>
+  - Deposit<pool>
+  - CurveTokenV1/V2
+3. Liquidity Gauges: stake LP get curve governance token?
+  - LiquidityGauge
+  - LiquidityGaugeV1/V2
+  - LiquidityGaugeReward
+4. DAO stuff
+5..? Other stuff, haven't decided if important
+"""
 CURVE_BASE_POOLS = [
     ClassifierSpec(
         abi_name="CurveTokenV1",
@@ -71,171 +72,101 @@ CURVE_BASE_POOLS = [
         abi_name="StableSwap3Pool",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xbEbc44782C7dB0a1A60Cb6fe97d0b483032FF1C7"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapAAVE",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xDeBF20617708857ebe4F679508E7b7863a8A8EeE"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapAETH",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xA96A65c051bF88B4095Ee1f2451C2A9d43F53Ae2"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapBUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x79a8C46DeA5aDa233ABaFFD40F3A0A2B1e5A4F27"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapCompound",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xA2B47E3D5c44877cca798226B7B8118F9BFb7A56"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapEURS",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x0Ce6a5fF5217e38315f87032CF90686C96627CAA"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwaphBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x4CA9b3063Ec5866A4B82E437059D2C43d1be596F"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapIronBank",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x2dded6Da1BF5DBdF597C45fcFaa3194e53EcfeAF"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapLink",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xf178c0b5bb7e7abf4e12a4838c7b7c5ba2c623c0"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapPAX",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x06364f10B501e868329afBc005b3492902d6C763"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwaprenBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x93054188d876f558f4a66B2EF1d97d16eDf0895B"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwaprETH",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xF9440930043eb3997fc70e1339dBb11F341de7A8"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapsAAVE",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xEB16Ae0052ed37f479f7fe63849198Df1765a733"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapsBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x7fC77b5c7614E1533320Ea6DDc2Eb61fa00A9714"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapsETH",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xc5424B857f758E906013F3555Dad202e4bdB4567"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapstETH",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xDC24316b9AE028F1497c275EB9192a3Ea0f67022"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapsUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xA5407eAE9Ba41422680e2e00537571bcC53efBfD"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapUSDT",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x52EA46506B9CC5Ef470C5bf89f17Dc28bB35D85C"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapY",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x45F783CCE6B7FF23B2ab2D70e416cdb7D6055f51"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapYv2",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x8925D9d9B4569D737a48499DeF3f67BaA5a144b9"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="DepositBUSD",
@@ -369,91 +300,51 @@ CURVE_META_POOLS = [
         abi_name="StableSwapbBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x071c661B4DeefB59E2a3DdB20Db036821eeE8F4b"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapDUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x8038C01A0390a8c547446a0b2c18fc9aEFEcc10c"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapGUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x4f062658EaAF2C1ccf8C8e36D6824CDf41167956"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapHUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x3eF6A01A0f81D6046290f3e2A8c5b843e738E604"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapLinkUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xE7a24EF0C5e95Ffb0f6684b813A78F2a3AD7D171"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapMUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x8474DdbE98F5aA3179B3B3F5942D724aFcdec9f6"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapoBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xd81dA8D904b52208541Bade1bD6595D8a251F8dd"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwappBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x7F55DDe206dbAD629C080068923b36fe9D6bDBeF"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapRSV",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xC18cC39da8b11dA8c3541C598eE022258F9744da"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwaptBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xC25099792E9349C7DD09759744ea681C7de2cb66"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapUSD",
@@ -462,29 +353,82 @@ CURVE_META_POOLS = [
             "0x3E01dD8a5E1fb3481F0F589056b428Fc308AF0Fb",
             "0x0f9cb53Ebe405d49A0bbdBD291A65Ff571bC83e1",
         ],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapUSDP",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x42d7025938bEc20B69cBae5A77421082407f053A"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
     ClassifierSpec(
         abi_name="StableSwapUST",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x890f4e345B1dAED0367A877a1612f86A1f86985f"],
-        classifiers={
-            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
-            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
-        },
     ),
 ]
 
+"""
+CURVE_LIQUIDITY_GAUGES = [
+    ClassifierSpec(
+        abi_name="LiquidityGauge",
+        protocol=Protocol.curve,
+        valid_contract_addresses=[
+            "0xbFcF63294aD7105dEa65aA58F8AE5BE2D9d0952A",  # 3Pool
+            "0x69Fb7c45726cfE2baDeE8317005d3F94bE838840",  # BUSD
+            "0x7ca5b0a2910B33e9759DC7dDB0413949071D7575",  # Compound
+            "0xC5cfaDA84E902aD92DD40194f0883ad49639b023",  # GUSD
+            "0x4c18E409Dc8619bFb6a1cB56D114C3f592E0aE79",  # hBTC
+            "0x2db0E83599a91b508Ac268a6197b8B14F5e72840",  # HUSD
+            "0x64E3C23bfc40722d3B649844055F1D51c1ac041d",  # PAX
+            "0xB1F2cdeC61db658F091671F5f199635aEF202CAC",  # renBTC
+            "0xC2b1DF84112619D190193E48148000e3990Bf627",  # USDK
+            "0xF98450B5602fa59CC66e1379DFfB6FDDc724CfC4",  # USDN
+            "0xBC89cd85491d81C6AD2954E6d0362Ee29fCa8F53",  # USDT
+            "0xFA712EE4788C042e2B7BB55E6cb8ec569C4530c1",  # Y
+        ],
+    ),
+    ClassifierSpec(
+        abi_name="LiquidityGaugeV2",
+        protocol=Protocol.curve,
+        valid_contract_addresses=[
+            "0xd662908ADA2Ea1916B3318327A97eB18aD588b5d",  # AAVE
+            "0x6d10ed2cF043E6fcf51A0e7b4C2Af3Fa06695707",  # ankrETH
+            "0xdFc7AdFa664b08767b735dE28f9E84cd30492aeE",  # bBTC
+            "0x90Bb609649E0451E5aD952683D64BD2d1f245840",  # EURS
+            "0x72e158d38dbd50a483501c24f792bdaaa3e7d55c",  # FRAX
+            "0x11137B10C210b579405c21A07489e28F3c040AB1",  # oBTC
+            "0xF5194c3325202F456c95c1Cf0cA36f8475C1949F",  # IronBank
+            "0xFD4D8a17df4C27c1dD245d153ccf4499e806C87D",  # Link
+            "0xd7d147c6Bb90A718c3De8C0568F9B560C79fa416",  # pBTC
+            "0x462253b8F74B72304c145DB0e4Eebd326B22ca39",  # sAAVE
+            "0x3C0FFFF15EA30C35d7A85B85c0782D6c94e1d238",  # sETH
+            "0x182B723a58739a9c974cFDB385ceaDb237453c28",  # stETH
+            "0x055be5DDB7A925BfEF3417FC157f53CA77cA7222",  # USDP
+            "0x3B7020743Bc2A4ca9EaF9D0722d42E20d6935855",  # UST
+            "0x8101E6760130be2C8Ace79643AB73500571b7162",  # Yv2
+        ],
+    ),
+    ClassifierSpec(
+        abi_name="LiquidityGaugeV3",
+        protocol=Protocol.curve,
+        valid_contract_addresses=[
+            "0x9582C4ADACB3BCE56Fea3e590F05c3ca2fb9C477",  # alUSD
+            "0x824F13f1a2F29cFEEa81154b46C0fc820677A637",  # rETH
+            "0x6955a55416a06839309018A8B0cB72c4DDC11f15",  # TriCrypto
+        ],
+    ),
+    ClassifierSpec(
+        abi_name="LiquidityGaugeReward",
+        protocol=Protocol.curve,
+        valid_contract_addresses=[
+            "0xAEA6c312f4b3E04D752946d329693F7293bC2e6D",  # DUSD
+            "0x5f626c30EC1215f4EdCc9982265E8b1F411D1352",  # MUSD
+            "0x4dC4A289a8E33600D8bD4cf5F6313E43a37adec7",  # RSV
+            "0x705350c4BcD35c9441419DdD5d2f097d7a55410F",  # sBTC
+            "0xA90996896660DEcC6E997655E065b23788857849",  # sUSDv2
+            "0x6828bcF74279eE32f2723eC536c22c51Eed383C6",  # tBTC
+        ],
+    ),
+]
+"""
+
 CURVE_CLASSIFIER_SPECS = [*CURVE_BASE_POOLS, *CURVE_META_POOLS]

mev_inspect/classifiers/specs/erc20.py

@@ -1,27 +1,15 @@
-from mev_inspect.schemas.classifiers import ClassifierSpec, TransferClassifier
-from mev_inspect.schemas.traces import DecodedCallTrace
-from mev_inspect.schemas.transfers import Transfer
-
-
-class ERC20TransferClassifier(TransferClassifier):
-    @staticmethod
-    def get_transfer(trace: DecodedCallTrace) -> Transfer:
-        return Transfer(
-            block_number=trace.block_number,
-            transaction_hash=trace.transaction_hash,
-            trace_address=trace.trace_address,
-            amount=trace.inputs["amount"],
-            to_address=trace.inputs["recipient"],
-            from_address=trace.inputs.get("sender", trace.from_address),
-            token_address=trace.to_address,
-        )
+from mev_inspect.schemas.classified_traces import (
+    Classification,
+    ClassifierSpec,
+)
 
 
 ERC20_SPEC = ClassifierSpec(
     abi_name="ERC20",
-    classifiers={
-        "transferFrom(address,address,uint256)": ERC20TransferClassifier,
-        "transfer(address,uint256)": ERC20TransferClassifier,
+    classifications={
+        "transferFrom(address,address,uint256)": Classification.transfer,
+        "transfer(address,uint256)": Classification.transfer,
+        "burn(address)": Classification.burn,
     },
 )
 

mev_inspect/classifiers/specs/opensea.py

@@ -1,42 +0,0 @@
-from typing import List, Optional
-
-from mev_inspect.classifiers.helpers import create_nft_trade_from_transfers
-from mev_inspect.schemas.classifiers import ClassifierSpec, NftTradeClassifier
-from mev_inspect.schemas.nft_trades import NftTrade
-from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
-from mev_inspect.schemas.transfers import Transfer
-
-OPENSEA_WALLET_ADDRESS = "0x5b3256965e7c3cf26e11fcaf296dfc8807c01073"
-
-
-class OpenseaClassifier(NftTradeClassifier):
-    @staticmethod
-    def parse_trade(
-        trace: DecodedCallTrace,
-        child_transfers: List[Transfer],
-    ) -> Optional[NftTrade]:
-        addresses = trace.inputs["addrs"]
-        buy_maker = addresses[1]
-        sell_maker = addresses[8]
-        target = addresses[4]
-
-        return create_nft_trade_from_transfers(
-            trace,
-            child_transfers,
-            collection_address=target,
-            seller_address=sell_maker,
-            buyer_address=buy_maker,
-            exchange_wallet_address=OPENSEA_WALLET_ADDRESS,
-        )
-
-
-OPENSEA_SPEC = ClassifierSpec(
-    abi_name="WyvernExchange",
-    protocol=Protocol.opensea,
-    valid_contract_addresses=["0x7be8076f4ea4a4ad08075c2508e481d6c946d12b"],
-    classifiers={
-        "atomicMatch_(address[14],uint256[18],uint8[8],bytes,bytes,bytes,bytes,bytes,bytes,uint8[2],bytes32[5])": OpenseaClassifier,
-    },
-)
-
-OPENSEA_CLASSIFIER_SPECS = [OPENSEA_SPEC]