diff --git a/.dockerignore b/.dockerignore index e182bb7..06cf653 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1 @@ -tests cache diff --git a/.env b/.env deleted file mode 100644 index 5b03ce8..0000000 --- a/.env +++ /dev/null @@ -1,8 +0,0 @@ -# Postgres -POSTGRES_SERVER=db -POSTGRES_USER=postgres -POSTGRES_PASSWORD=password -POSTGRES_DB=mev_inspect - -# SQLAlchemy -SQLALCHEMY_DATABASE_URI=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_SERVER/$POSTGRES_DB diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 4211a12..10f71db 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -55,4 +55,4 @@ jobs: - name: Test with pytest shell: bash - run: poetry run test + run: poetry run pytest --cov=mev_inspect tests diff --git a/README.md b/README.md index 35b9b80..dc42c17 100644 --- a/README.md +++ b/README.md @@ -1,104 +1,143 @@ -# mev-inspect -A [WIP] Ethereum MEV Inspector in Python managed by Poetry +# mev-inspect-py +> illuminating the dark forest 🌲🔦 -## Containers -mev-inspect's local setup is built on [Docker Compose](https://docs.docker.com/compose/) +**mev-inspect-py** is an MEV inspector for Ethereum -By default it starts up: -- `mev-inspect` - a container with the code in this repo used for running scripts -- `db` - a postgres database instance -- `pgadmin` - a postgres DB UI for querying and more (avaiable at localhost:5050) +Given a block, mev-inspect finds: +- miner payments (gas + coinbase) +- tokens transfers and profit +- swaps and [arbitrages](https://twitter.com/bertcmiller/status/142763202826305946://twitter.com/bertcmiller/status/1427632028263059462) +- ...and more + +Data is stored in Postgres for analysis ## Running locally -Setup [Docker](https://www.docker.com/products/docker-desktop) -Setup [Poetry](https://python-poetry.org/docs/#osx--linux--bashonwindows-install-instructions) +mev-inspect-py is built to run on kubernetes locally and in production -Install dependencies through poetry +### Install dependencies + +1. Setup a local kubernetes deployment (we use [kind](https://kind.sigs.k8s.io/docs/user/quick-start)) + +2. Setup [Tilt](https://docs.tilt.dev/install.html) which manages the local deployment + +### Start up + +Set an environment variable `RPC_URL` to an RPC for fetching blocks +Example: +``` +export RPC_URL="http://111.111.111.111:8546" +``` + +**Note: mev-inspect-py currently requires and RPC with support for parity traces** + +Next, start all servcies with: +``` +tilt up +``` + +Press "space" to see a browser of the services starting up + +On first startup, you'll need to apply database migrations. Apply with: +``` +kubectl exec deploy/mev-inspect-deployment -- alembic upgrade head +``` + +## Inspecting + +### Inspect a single block + +Inspecting block [12914944](https://twitter.com/mevalphaleak/status/1420416437575901185) +``` +kubectl exec deploy/mev-inspect-deployment -- poetry run inspect-block 12914944 +``` + +### Inspect many blocks + +Inspecting blocks 12914944 to 12914954 +``` +kubectl exec deploy/mev-inspect-deployment -- poetry run inspect-many-blocks 12914944 12914954 +``` + +### Inspect all incoming blocks + +Start a block listener with +``` +kubectl exec deploy/mev-inspect-deployment -- /app/listener start +``` + +By default, it will pick up wherever you left off. +If running for the first time, listener starts at the latest block + +See logs for the listener with +``` +kubectl exec deploy/mev-inspect-deployment -- tail -f listener.log +``` + +And stop the listener with +``` +kubectl exec deploy/mev-inspect-deployment -- /app/listener stop +``` + +## Contributing + +### Guide + +✨ Coming soon + +### Pre-commit + +We use pre-commit to maintain a consistent style, prevent errors, and ensure test coverage. + +To set up, install dependencies through poetry ``` poetry install ``` -Start the services (optionally as daemon) -``` -poetry run start [-d] -``` - -Apply the latest migrations against the local DB: -``` -poetry run exec alembic upgrade head -``` - -Run inspect on a block -``` -poetry run inspect -b/--block-number 11931270 -r/--rpc 'http://111.11.11.111:8545/' -``` - -To stop the services (if running in the background, otherwise just ctrl+c) -``` -poetry run stop -``` - -MEV container can be attached via -``` -poetry run attach -``` - -Running additional compose commands are possible through standard `docker -compose ...` calls. Check `docker compose help` for more tools available - -## Executing scripts -Any script can be run from the mev-inspect container like -``` -poetry run exec -``` - -For example -``` -poetry run exec python examples/uniswap_inspect.py -block_number=123 -rpc='111.111.111' -``` - -### Poetry Scripts -```bash -# code check -poetry run lint # linting via Pylint -poetry run test # testing and code coverage with Pytest -poetry run isort # fixing imports -poetry run mypy # type checking -poetry run black # style guide -poetry run pre-commit run --all-files # runs Black, PyLint and MyPy -# docker management -poetry run start [-d] # starts all services, optionally as a daemon -poetry run stop # shutsdown all services or just ctrl + c if foreground -poetry run build # rebuilds containers -poetry run attach # enters the mev-inspect container in interactive mode -# launches inspection script -poetry run inspect -b/--block-number 11931270 -r/--rpc 'http://111.11.11.111:8545/' -``` - - -## Rebuilding containers -After changes to the app's Dockerfile, rebuild with -``` -poetry run build -``` - -## Using PGAdmin - -1. Go to [localhost:5050](localhost:5050) - -2. Login with the PGAdmin username and password in `.env` - -3. Add a new engine for mev_inspect with - - host: db - - user / password: see `.env` - -## Contributing - -Pre-commit is used to maintain a consistent style, prevent errors and ensure test coverage. - -Install pre-commit with: +Then install pre-commit hooks with ``` poetry run pre-commit install ``` -Update README if needed +### Tests + +Run tests with +``` +kubectl exec deploy/mev-inspect-deployment -- poetry run pytest --cov=mev_inspect tests +``` + +## FAQ + +### How do I delete / reset my local postgres data? + +Stop the system if running +``` +tilt down +``` + +Delete it with +``` +kubectl delete pvc data-postgresql-postgresql-0 +``` + +Start back up again +``` +tilt up +``` + +And rerun migrations to create the tables again +``` +kubectl exec deploy/mev-inspect-deployment -- alembic upgrade head +``` + +### I was using the docker-compose setup and want to switch to kube, now what? + +Re-add the old `docker-compose.yml` file to your mev-inspect-py directory + +A copy can be found [here](https://github.com/flashbots/mev-inspect-py/blob/ef60c097719629a7d2dc56c6e6c9a100fb706f76/docker-compose.yml) + +Tear down docker-compose resources +``` +docker compose down +``` + +Then go through the steps in the current README for kube setup diff --git a/scripts/inspect_commands.py b/cli.py similarity index 86% rename from scripts/inspect_commands.py rename to cli.py index ceb2138..631de8d 100644 --- a/scripts/inspect_commands.py +++ b/cli.py @@ -1,3 +1,5 @@ +import os + import click from web3 import Web3 @@ -6,6 +8,9 @@ from mev_inspect.inspect_block import inspect_block from mev_inspect.provider import get_base_provider +RPC_URL_ENV = "RPC_URL" + + @click.group() def cli(): pass @@ -13,7 +18,7 @@ def cli(): @cli.command() @click.argument("block_number", type=int) -@click.argument("rpc") +@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, "")) @click.option("--cache/--no-cache", default=True) def inspect_block_command(block_number: int, rpc: str, cache: bool): db_session = get_session() @@ -29,7 +34,7 @@ def inspect_block_command(block_number: int, rpc: str, cache: bool): @cli.command() @click.argument("after_block", type=int) @click.argument("before_block", type=int) -@click.argument("rpc") +@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, "")) @click.option("--cache/--no-cache", default=True) def inspect_many_blocks_command( after_block: int, before_block: int, rpc: str, cache: bool @@ -61,5 +66,9 @@ def inspect_many_blocks_command( ) +def get_rpc_url() -> str: + return os.environ["RPC_URL"] + + if __name__ == "__main__": cli() diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index c58a3ad..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -services: - mev-inspect: - build: . - depends_on: - - db - env_file: - - .env - volumes: - - .:/app - tty: true - - db: - image: postgres:12 - volumes: - - mev-inspect-db-data:/var/lib/postgresql/data/pgdata - env_file: - - .env - environment: - - PGDATA=/var/lib/postgresql/data/pgdata - ports: - - 5432:5432 - -volumes: - mev-inspect-db-data: diff --git a/mev_inspect/config.ini b/mev_inspect/config.ini deleted file mode 100644 index 83863a9..0000000 --- a/mev_inspect/config.ini +++ /dev/null @@ -1,7 +0,0 @@ -[RPC] -Endpoint = http://localhost:8545/ - -[ADDRESSES] -UniswapV2Router = 0x7a250d5630B4cF539739dF2C5dAcb4c659F2488D -SushiswapV2Router = 0xd9e1cE17f2641f24aE83637ab66a2cca9C378B9F -WETH = 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 diff --git a/mev_inspect/config.py b/mev_inspect/config.py deleted file mode 100644 index 465a098..0000000 --- a/mev_inspect/config.py +++ /dev/null @@ -1,13 +0,0 @@ -import os -import configparser - - -THIS_FILE_DIRECTORY = os.path.dirname(__file__) -CONFIG_PATH = os.path.join(THIS_FILE_DIRECTORY, "config.ini") - - -def load_config(): - config = configparser.ConfigParser() - config.read(CONFIG_PATH) - - return config diff --git a/mev_inspect/tokenflow.py b/mev_inspect/tokenflow.py index 7350bdb..4af2092 100644 --- a/mev_inspect/tokenflow.py +++ b/mev_inspect/tokenflow.py @@ -1,13 +1,8 @@ from typing import List, Optional -from mev_inspect.config import load_config from mev_inspect.schemas import Block, Trace, TraceType -config = load_config() - -rpc_url = config["RPC"]["Endpoint"] -weth_address = config["ADDRESSES"]["WETH"] -# w3 = Web3(HTTPProvider(rpc_url)) +weth_address = "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2" cache_directory = "./cache" diff --git a/pyproject.toml b/pyproject.toml index 47b50a3..26607a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,20 +29,8 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] -lint = 'scripts.poetry.dev_tools:lint' -test = 'scripts.poetry.dev_tools:test' -isort = 'scripts.poetry.dev_tools:isort' -black = 'scripts.poetry.dev_tools:black' -pre_commit = 'scripts.poetry.dev_tools:pre_commit' -start = 'scripts.poetry.docker:start' -stop = 'scripts.poetry.docker:stop' -build = 'scripts.poetry.docker:build' -attach = 'scripts.poetry.docker:attach' -exec = 'scripts.poetry.docker:exec' -inspect = 'scripts.poetry.inspect:inspect' -inspect-many = 'scripts.poetry.inspect:inspect_many' -inspect-block = 'scripts.inspect_commands:inspect_block_command' -inspect-many-blocks = 'scripts.inspect_commands:inspect_many_blocks_command' +inspect-block = 'cli:inspect_block_command' +inspect-many-blocks = 'cli:inspect_many_blocks_command' [tool.black] exclude = ''' diff --git a/scripts/poetry/dev_tools.py b/scripts/poetry/dev_tools.py deleted file mode 100644 index 1a5a95e..0000000 --- a/scripts/poetry/dev_tools.py +++ /dev/null @@ -1,34 +0,0 @@ -from subprocess import check_call -import click - - -def lint(): - check_call(["pylint", "."]) - - -def test(): - check_call(["pytest", "--cov=mev_inspect", "tests"]) - - -@click.command() -@click.option("-c", required=False, is_flag=True) -def isort(c: str): - """if c is present run isort in diff mode""" - if c: - check_call(["isort", "."]) - else: - check_call(["isort", "--diff", "."]) - - -def mypy(): - check_call(["mypy", "."]) - - -@click.command() -@click.option("-c", required=False, is_flag=True) -def black(c: str = None): - """if c is present run black in diff mode""" - if c: - check_call(["black", "--diff", "--color", "."]) - else: - check_call(["black", "."]) diff --git a/scripts/poetry/docker.py b/scripts/poetry/docker.py deleted file mode 100644 index 31f0579..0000000 --- a/scripts/poetry/docker.py +++ /dev/null @@ -1,33 +0,0 @@ -from subprocess import check_call -from typing import List - -import click - - -@click.command() -@click.option("-d", required=False, is_flag=True) -def start(d: str): - """if d is present, run docker compose as daemon""" - if d: - check_call(["docker-compose", "up", "-d"]) - click.echo("docker running in the background...") - else: - check_call(["docker-compose", "up"]) - - -def stop(): - check_call(["docker-compose", "down"]) - - -def build(): - check_call(["docker-compose", "build"]) - - -def attach(): - check_call(["docker", "exec", "-it", "mev-inspect-py_mev-inspect_1", "bash"]) - - -@click.command() -@click.argument("args", nargs=-1) -def exec(args: List[str]): - check_call(["docker-compose", "exec", "mev-inspect", *args]) diff --git a/scripts/poetry/inspect.py b/scripts/poetry/inspect.py deleted file mode 100644 index df6979b..0000000 --- a/scripts/poetry/inspect.py +++ /dev/null @@ -1,59 +0,0 @@ -from subprocess import check_call - -import click - - -@click.command() -@click.option( - "-b", "--block-number", type=str, help="the block number you are targetting" -) -@click.option( - "-r", "--rpc", help="rpc endpoint, this needs to have parity style traces" -) -@click.option( - "--cache/--no-cache", - help="whether to read / write to the cache", - default=True, -) -def inspect(block_number: str, rpc: str, cache: bool): - check_call( - [ - "docker", - "compose", - "exec", - "mev-inspect", - "python", - "./scripts/inspect_block.py", - "inspect-block", - block_number, - rpc, - "--cache" if cache else "--no-cache", - ] - ) - - -@click.command() -@click.argument("after_block", type=str) -@click.argument("before_block", type=str) -@click.argument("rpc") -@click.option( - "--cache/--no-cache", - help="whether to read / write to the cache", - default=True, -) -def inspect_many(after_block: str, before_block: str, rpc: str, cache: bool): - check_call( - [ - "docker", - "compose", - "exec", - "mev-inspect", - "python", - "./scripts/inspect_block.py", - "inspect-many-blocks", - after_block, - before_block, - rpc, - "--cache" if cache else "--no-cache", - ] - )