From be9ae86d5cdd241eedf60c62899c2ecb090e0948 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 12:54:08 -0400 Subject: [PATCH 01/13] Update README for Kube --- README.md | 200 +++++++++++++++++++++++++++++------------------------- 1 file changed, 109 insertions(+), 91 deletions(-) diff --git a/README.md b/README.md index 35b9b80..06d9a9c 100644 --- a/README.md +++ b/README.md @@ -1,104 +1,122 @@ -# mev-inspect -A [WIP] Ethereum MEV Inspector in Python managed by Poetry +# mev-inspect-py +> illuminating the dark forest 🌲🔦 -## Containers -mev-inspect's local setup is built on [Docker Compose](https://docs.docker.com/compose/) +**mev-inspect-py** is an MEV inspector for Ethereum -By default it starts up: -- `mev-inspect` - a container with the code in this repo used for running scripts -- `db` - a postgres database instance -- `pgadmin` - a postgres DB UI for querying and more (avaiable at localhost:5050) +Given a block, mev-inspect finds: +- miner payments (gas + coinbase) +- tokens transfers and profit +- swaps and [arbitrages](https://twitter.com/bertcmiller/status/142763202826305946://twitter.com/bertcmiller/status/1427632028263059462) +- ...and more + +Data is stored in Postgres for analysis ## Running locally -Setup [Docker](https://www.docker.com/products/docker-desktop) -Setup [Poetry](https://python-poetry.org/docs/#osx--linux--bashonwindows-install-instructions) +mev-inspect-py is built to run on kubernetes locally and in production -Install dependencies through poetry +### Install dependencies + +Setup a local kubernetes deployment (we use [kind](https://kind.sigs.k8s.io/docs/user/quick-start)) +Setup [Tilt](https://docs.tilt.dev/install.html) which manages the local deployment + +### Start up + +Set an environment variable `RPC_URL` to an RPC for fetching blocks +Example: +``` +export RPC_URL="http://111.111.111.111:8546" +``` + +Note: mev-inspect-py currently requires and RPC with support for parity traces + +Next, start all servcies with: +``` +tilt up +``` + +Press "space" to see a browser of the services starting up + +On first startup, you'll need to apply database migrations. Apply with: +``` +kubectl exec deploy/mev-inspect-deployment -- alembic upgrade head +``` + +## Inspecting + +### Inspect a single block + +Inspecting block 12345 +``` +kubectl exec deploy/mev-inspect-deployment -- poetry run inspect-block 12345 +``` + +### Inspect many blocks + +Inspecting blocks 12345 to 13345 +``` +kubectl exec deploy/mev-inspect-deployment -- poetry run inspect-many-blocks 12345 13345 +``` + +### Inspect all incoming blocks + +Start a block listener with +``` +kubectl exec deploy/mev-inspect-deployment -- /app/listener start +``` + +By default, it will pick up wherever you left off. +If running for the first time, listener starts at the latest block + +See logs for the listener with +``` +kubectl exec deploy/mev-inspect-deployment -- tail -f listener.log +``` + +And stop the listener with +``` +kubectl exec deploy/mev-inspect-deployment -- /app/listener stop +``` + +## Contributing + +### Guide + +Coming soon + +### Pre-commit + +We use pre-commit to maintain a consistent style, prevent errors, and ensure test coverage. + +To set up, install dependencies through poetry ``` poetry install ``` -Start the services (optionally as daemon) -``` -poetry run start [-d] -``` - -Apply the latest migrations against the local DB: -``` -poetry run exec alembic upgrade head -``` - -Run inspect on a block -``` -poetry run inspect -b/--block-number 11931270 -r/--rpc 'http://111.11.11.111:8545/' -``` - -To stop the services (if running in the background, otherwise just ctrl+c) -``` -poetry run stop -``` - -MEV container can be attached via -``` -poetry run attach -``` - -Running additional compose commands are possible through standard `docker -compose ...` calls. Check `docker compose help` for more tools available - -## Executing scripts -Any script can be run from the mev-inspect container like -``` -poetry run exec -``` - -For example -``` -poetry run exec python examples/uniswap_inspect.py -block_number=123 -rpc='111.111.111' -``` - -### Poetry Scripts -```bash -# code check -poetry run lint # linting via Pylint -poetry run test # testing and code coverage with Pytest -poetry run isort # fixing imports -poetry run mypy # type checking -poetry run black # style guide -poetry run pre-commit run --all-files # runs Black, PyLint and MyPy -# docker management -poetry run start [-d] # starts all services, optionally as a daemon -poetry run stop # shutsdown all services or just ctrl + c if foreground -poetry run build # rebuilds containers -poetry run attach # enters the mev-inspect container in interactive mode -# launches inspection script -poetry run inspect -b/--block-number 11931270 -r/--rpc 'http://111.11.11.111:8545/' -``` - - -## Rebuilding containers -After changes to the app's Dockerfile, rebuild with -``` -poetry run build -``` - -## Using PGAdmin - -1. Go to [localhost:5050](localhost:5050) - -2. Login with the PGAdmin username and password in `.env` - -3. Add a new engine for mev_inspect with - - host: db - - user / password: see `.env` - -## Contributing - -Pre-commit is used to maintain a consistent style, prevent errors and ensure test coverage. - -Install pre-commit with: +Then install pre-commit hooks with ``` poetry run pre-commit install ``` -Update README if needed + +## FAQ + +### How do I delete / reset my local postgres data? + +Stop the system if running +``` +tilt down +``` + +Then delete with +``` +kubectl delete pvc data-postgresql-postgresql-0 +``` + +### I was using the docker-compose setup and want to switch to kube, now what? + +Make sure the docker-compose resources are down +``` +docker compose down +``` + +Then go through the steps in the current README for kube setup From 663a97e84fbaebc800c9e19e73fa8c34f9c075d0 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 12:56:17 -0400 Subject: [PATCH 02/13] Fix spacing --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 06d9a9c..b7fabbd 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,9 @@ mev-inspect-py is built to run on kubernetes locally and in production ### Install dependencies -Setup a local kubernetes deployment (we use [kind](https://kind.sigs.k8s.io/docs/user/quick-start)) -Setup [Tilt](https://docs.tilt.dev/install.html) which manages the local deployment +1. Setup a local kubernetes deployment (we use [kind](https://kind.sigs.k8s.io/docs/user/quick-start)) + +2. Setup [Tilt](https://docs.tilt.dev/install.html) which manages the local deployment ### Start up From 036228036d62d85e69239b53f798bc141c7f3f82 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 12:57:28 -0400 Subject: [PATCH 03/13] Bold parity --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b7fabbd..892960c 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Example: export RPC_URL="http://111.111.111.111:8546" ``` -Note: mev-inspect-py currently requires and RPC with support for parity traces +**Note: mev-inspect-py currently requires and RPC with support for parity traces** Next, start all servcies with: ``` From 4d5c8977c180d1e82323de42e5aec36a134812e9 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 13:04:37 -0400 Subject: [PATCH 04/13] Remove docker compose and old poetry scripts --- scripts/inspect_commands.py => cli.py | 13 +++++- docker-compose.yml | 24 ----------- pyproject.toml | 16 +------- scripts/poetry/dev_tools.py | 34 --------------- scripts/poetry/docker.py | 33 --------------- scripts/poetry/inspect.py | 59 --------------------------- 6 files changed, 13 insertions(+), 166 deletions(-) rename scripts/inspect_commands.py => cli.py (86%) delete mode 100644 docker-compose.yml delete mode 100644 scripts/poetry/dev_tools.py delete mode 100644 scripts/poetry/docker.py delete mode 100644 scripts/poetry/inspect.py diff --git a/scripts/inspect_commands.py b/cli.py similarity index 86% rename from scripts/inspect_commands.py rename to cli.py index ceb2138..631de8d 100644 --- a/scripts/inspect_commands.py +++ b/cli.py @@ -1,3 +1,5 @@ +import os + import click from web3 import Web3 @@ -6,6 +8,9 @@ from mev_inspect.inspect_block import inspect_block from mev_inspect.provider import get_base_provider +RPC_URL_ENV = "RPC_URL" + + @click.group() def cli(): pass @@ -13,7 +18,7 @@ def cli(): @cli.command() @click.argument("block_number", type=int) -@click.argument("rpc") +@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, "")) @click.option("--cache/--no-cache", default=True) def inspect_block_command(block_number: int, rpc: str, cache: bool): db_session = get_session() @@ -29,7 +34,7 @@ def inspect_block_command(block_number: int, rpc: str, cache: bool): @cli.command() @click.argument("after_block", type=int) @click.argument("before_block", type=int) -@click.argument("rpc") +@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, "")) @click.option("--cache/--no-cache", default=True) def inspect_many_blocks_command( after_block: int, before_block: int, rpc: str, cache: bool @@ -61,5 +66,9 @@ def inspect_many_blocks_command( ) +def get_rpc_url() -> str: + return os.environ["RPC_URL"] + + if __name__ == "__main__": cli() diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index c58a3ad..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -services: - mev-inspect: - build: . - depends_on: - - db - env_file: - - .env - volumes: - - .:/app - tty: true - - db: - image: postgres:12 - volumes: - - mev-inspect-db-data:/var/lib/postgresql/data/pgdata - env_file: - - .env - environment: - - PGDATA=/var/lib/postgresql/data/pgdata - ports: - - 5432:5432 - -volumes: - mev-inspect-db-data: diff --git a/pyproject.toml b/pyproject.toml index 47b50a3..26607a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,20 +29,8 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] -lint = 'scripts.poetry.dev_tools:lint' -test = 'scripts.poetry.dev_tools:test' -isort = 'scripts.poetry.dev_tools:isort' -black = 'scripts.poetry.dev_tools:black' -pre_commit = 'scripts.poetry.dev_tools:pre_commit' -start = 'scripts.poetry.docker:start' -stop = 'scripts.poetry.docker:stop' -build = 'scripts.poetry.docker:build' -attach = 'scripts.poetry.docker:attach' -exec = 'scripts.poetry.docker:exec' -inspect = 'scripts.poetry.inspect:inspect' -inspect-many = 'scripts.poetry.inspect:inspect_many' -inspect-block = 'scripts.inspect_commands:inspect_block_command' -inspect-many-blocks = 'scripts.inspect_commands:inspect_many_blocks_command' +inspect-block = 'cli:inspect_block_command' +inspect-many-blocks = 'cli:inspect_many_blocks_command' [tool.black] exclude = ''' diff --git a/scripts/poetry/dev_tools.py b/scripts/poetry/dev_tools.py deleted file mode 100644 index 1a5a95e..0000000 --- a/scripts/poetry/dev_tools.py +++ /dev/null @@ -1,34 +0,0 @@ -from subprocess import check_call -import click - - -def lint(): - check_call(["pylint", "."]) - - -def test(): - check_call(["pytest", "--cov=mev_inspect", "tests"]) - - -@click.command() -@click.option("-c", required=False, is_flag=True) -def isort(c: str): - """if c is present run isort in diff mode""" - if c: - check_call(["isort", "."]) - else: - check_call(["isort", "--diff", "."]) - - -def mypy(): - check_call(["mypy", "."]) - - -@click.command() -@click.option("-c", required=False, is_flag=True) -def black(c: str = None): - """if c is present run black in diff mode""" - if c: - check_call(["black", "--diff", "--color", "."]) - else: - check_call(["black", "."]) diff --git a/scripts/poetry/docker.py b/scripts/poetry/docker.py deleted file mode 100644 index 31f0579..0000000 --- a/scripts/poetry/docker.py +++ /dev/null @@ -1,33 +0,0 @@ -from subprocess import check_call -from typing import List - -import click - - -@click.command() -@click.option("-d", required=False, is_flag=True) -def start(d: str): - """if d is present, run docker compose as daemon""" - if d: - check_call(["docker-compose", "up", "-d"]) - click.echo("docker running in the background...") - else: - check_call(["docker-compose", "up"]) - - -def stop(): - check_call(["docker-compose", "down"]) - - -def build(): - check_call(["docker-compose", "build"]) - - -def attach(): - check_call(["docker", "exec", "-it", "mev-inspect-py_mev-inspect_1", "bash"]) - - -@click.command() -@click.argument("args", nargs=-1) -def exec(args: List[str]): - check_call(["docker-compose", "exec", "mev-inspect", *args]) diff --git a/scripts/poetry/inspect.py b/scripts/poetry/inspect.py deleted file mode 100644 index df6979b..0000000 --- a/scripts/poetry/inspect.py +++ /dev/null @@ -1,59 +0,0 @@ -from subprocess import check_call - -import click - - -@click.command() -@click.option( - "-b", "--block-number", type=str, help="the block number you are targetting" -) -@click.option( - "-r", "--rpc", help="rpc endpoint, this needs to have parity style traces" -) -@click.option( - "--cache/--no-cache", - help="whether to read / write to the cache", - default=True, -) -def inspect(block_number: str, rpc: str, cache: bool): - check_call( - [ - "docker", - "compose", - "exec", - "mev-inspect", - "python", - "./scripts/inspect_block.py", - "inspect-block", - block_number, - rpc, - "--cache" if cache else "--no-cache", - ] - ) - - -@click.command() -@click.argument("after_block", type=str) -@click.argument("before_block", type=str) -@click.argument("rpc") -@click.option( - "--cache/--no-cache", - help="whether to read / write to the cache", - default=True, -) -def inspect_many(after_block: str, before_block: str, rpc: str, cache: bool): - check_call( - [ - "docker", - "compose", - "exec", - "mev-inspect", - "python", - "./scripts/inspect_block.py", - "inspect-many-blocks", - after_block, - before_block, - rpc, - "--cache" if cache else "--no-cache", - ] - ) From 37cf615c75dd6f470ad2555847c2a88bab354a3b Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 13:06:33 -0400 Subject: [PATCH 05/13] Update README for more interesting blocks --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 892960c..c8aeda4 100644 --- a/README.md +++ b/README.md @@ -46,16 +46,16 @@ kubectl exec deploy/mev-inspect-deployment -- alembic upgrade head ### Inspect a single block -Inspecting block 12345 +Inspecting block [12914944](https://twitter.com/mevalphaleak/status/1420416437575901185) ``` -kubectl exec deploy/mev-inspect-deployment -- poetry run inspect-block 12345 +kubectl exec deploy/mev-inspect-deployment -- poetry run inspect-block 12914944 ``` ### Inspect many blocks -Inspecting blocks 12345 to 13345 +Inspecting blocks 12914944 to 12914954 ``` -kubectl exec deploy/mev-inspect-deployment -- poetry run inspect-many-blocks 12345 13345 +kubectl exec deploy/mev-inspect-deployment -- poetry run inspect-many-blocks 12914944 12914954 ``` ### Inspect all incoming blocks From 52679cd3cccd7fba67590c3509e2d66867ed925c Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 13:09:14 -0400 Subject: [PATCH 06/13] Update actions to call tests directly --- .github/workflows/github-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 4211a12..10f71db 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -55,4 +55,4 @@ jobs: - name: Test with pytest shell: bash - run: poetry run test + run: poetry run pytest --cov=mev_inspect tests From 0afc1494f17c3a58b53d0d917b31e4d2e9cf92fd Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 13:11:13 -0400 Subject: [PATCH 07/13] Add tests to README --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c8aeda4..421d501 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,12 @@ Then install pre-commit hooks with poetry run pre-commit install ``` +### Tests + +Run tests with +``` +kubectl exec deploy/mev-inspect-deployment -- poetry run pytest --cov=mev_inspect tests +``` ## FAQ From 7b036cc6205e0bc3be4c4fab9df80fb6b8a5d534 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 13:12:35 -0400 Subject: [PATCH 08/13] Add stars --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 421d501..39adb6b 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ kubectl exec deploy/mev-inspect-deployment -- /app/listener stop ### Guide -Coming soon +✨ Coming soon ### Pre-commit From 00c97ffe725a57f276f95cf9ca0fb6fa460742a4 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 13:15:18 -0400 Subject: [PATCH 09/13] Better instructions for switching from docker compose --- README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 39adb6b..dc42c17 100644 --- a/README.md +++ b/README.md @@ -114,14 +114,28 @@ Stop the system if running tilt down ``` -Then delete with +Delete it with ``` kubectl delete pvc data-postgresql-postgresql-0 ``` +Start back up again +``` +tilt up +``` + +And rerun migrations to create the tables again +``` +kubectl exec deploy/mev-inspect-deployment -- alembic upgrade head +``` + ### I was using the docker-compose setup and want to switch to kube, now what? -Make sure the docker-compose resources are down +Re-add the old `docker-compose.yml` file to your mev-inspect-py directory + +A copy can be found [here](https://github.com/flashbots/mev-inspect-py/blob/ef60c097719629a7d2dc56c6e6c9a100fb706f76/docker-compose.yml) + +Tear down docker-compose resources ``` docker compose down ``` From 86fdeddfaa6bf1165c943009fb490227e0a19e9b Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 13:21:24 -0400 Subject: [PATCH 10/13] Not used --- .env | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .env diff --git a/.env b/.env deleted file mode 100644 index 5b03ce8..0000000 --- a/.env +++ /dev/null @@ -1,8 +0,0 @@ -# Postgres -POSTGRES_SERVER=db -POSTGRES_USER=postgres -POSTGRES_PASSWORD=password -POSTGRES_DB=mev_inspect - -# SQLAlchemy -SQLALCHEMY_DATABASE_URI=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_SERVER/$POSTGRES_DB From 576f7dc507effad1144b4929d817c3b823049fd1 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 13:22:04 -0400 Subject: [PATCH 11/13] Don't ignore tests. Run tests int he container --- .dockerignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index e182bb7..06cf653 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1 @@ -tests cache From 747dc5dfe1cce4f3dabd90f5f0d383953dbf7bfa Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 13:25:21 -0400 Subject: [PATCH 12/13] Remove some old config files --- mev_inspect/config.ini | 7 ------- mev_inspect/config.py | 13 ------------- mev_inspect/tokenflow.py | 7 +------ 3 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 mev_inspect/config.ini delete mode 100644 mev_inspect/config.py diff --git a/mev_inspect/config.ini b/mev_inspect/config.ini deleted file mode 100644 index 83863a9..0000000 --- a/mev_inspect/config.ini +++ /dev/null @@ -1,7 +0,0 @@ -[RPC] -Endpoint = http://localhost:8545/ - -[ADDRESSES] -UniswapV2Router = 0x7a250d5630B4cF539739dF2C5dAcb4c659F2488D -SushiswapV2Router = 0xd9e1cE17f2641f24aE83637ab66a2cca9C378B9F -WETH = 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 diff --git a/mev_inspect/config.py b/mev_inspect/config.py deleted file mode 100644 index 465a098..0000000 --- a/mev_inspect/config.py +++ /dev/null @@ -1,13 +0,0 @@ -import os -import configparser - - -THIS_FILE_DIRECTORY = os.path.dirname(__file__) -CONFIG_PATH = os.path.join(THIS_FILE_DIRECTORY, "config.ini") - - -def load_config(): - config = configparser.ConfigParser() - config.read(CONFIG_PATH) - - return config diff --git a/mev_inspect/tokenflow.py b/mev_inspect/tokenflow.py index 7350bdb..ff61f1c 100644 --- a/mev_inspect/tokenflow.py +++ b/mev_inspect/tokenflow.py @@ -1,13 +1,8 @@ from typing import List, Optional -from mev_inspect.config import load_config from mev_inspect.schemas import Block, Trace, TraceType -config = load_config() - -rpc_url = config["RPC"]["Endpoint"] -weth_address = config["ADDRESSES"]["WETH"] -# w3 = Web3(HTTPProvider(rpc_url)) +weth_address = "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" cache_directory = "./cache" From 18e45ee4372f7c860d30b592b88e875ffdd43396 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 20 Sep 2021 13:28:16 -0400 Subject: [PATCH 13/13] Lower weth address --- mev_inspect/tokenflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mev_inspect/tokenflow.py b/mev_inspect/tokenflow.py index ff61f1c..4af2092 100644 --- a/mev_inspect/tokenflow.py +++ b/mev_inspect/tokenflow.py @@ -2,7 +2,7 @@ from typing import List, Optional from mev_inspect.schemas import Block, Trace, TraceType -weth_address = "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" +weth_address = "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2" cache_directory = "./cache"