Merge pull request #30 from pmdaly/add-poetry

Add poetry

.github/workflows/github-actions.yml

@@ -4,22 +4,55 @@ on: [push, pull_request]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+
       - name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
-      - name: Install dependencies
+
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
+
+      - name: Bootstrap poetry
+        shell: bash
         run: |
-          python -m pip install --upgrade pip
+          curl -sL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py \
-          pip install -r requirements.txt
+            | python - -y 
-          pip install -r requirements_dev.txt
+
+      - name: Update PATH
+        if: ${{ matrix.os != 'Windows' }}
+        shell: bash
+        run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Configure poetry
+        shell: bash
+        run: poetry config virtualenvs.in-project true
+
+      - name: Set up cache
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Ensure cache is healthy
+        if: steps.cache.outputs.cache-hit == 'true'
+        shell: bash
+        run: timeout 10s poetry run pip --version || rm -rf .venv
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
       - name: Run precommit
         run: |
-          pre-commit run --all-files
+          poetry run pre-commit
-      - name: Test with unittest
+
-        run: |
+      - name: Test with pytest
-          python -m unittest tests/*.py
+        shell: bash
+        run: poetry run test

.gitignore

@@ -3,5 +3,13 @@ env/
 __pycache__
 .mypy_cache
 
-*.swp
+# vim temp
+*.sw?
+.*.sw?
 
+# pytest cache
+.pytest_cache/
+
+# coverage
+htmlcov
+.coverage*

.python-version

@@ -1 +0,0 @@
-3.9.4

Dockerfile

@@ -1,10 +1,19 @@
 FROM python:3.9
 
-COPY ./requirements.txt /app/requirements.txt
+RUN pip install -U pip \
-RUN pip install -r /app/requirements.txt
+    && apt-get update \
+    && curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
+
+ENV PATH="${PATH}:/root/.poetry/bin"
 
 COPY . /app
 WORKDIR /app/
 
-ENTRYPOINT ["./run.sh"]
+# poetry uses virtual env by default, turn this off inside container
-CMD []
+RUN poetry config virtualenvs.create false && \
+    poetry install
+
+# easter eggs 😝
+RUN echo "PS1='🕵️:\[\033[1;36m\]\h \[\033[1;34m\]\W\[\033[0;35m\]\[\033[1;36m\]$ \[\033[0m\]'" >> ~/.bashrc
+
+CMD /bin/bash

README.md

@@ -1,5 +1,5 @@
 # mev-inspect
-A [WIP] Ethereum MEV Inspector in Python
+A [WIP] Ethereum MEV Inspector in Python managed by Poetry
 
 ## Containers
 mev-inspect's local setup is built on [Docker Compose](https://docs.docker.com/compose/)
@@ -12,44 +12,64 @@ By default it starts up:
 ## Running locally
 Setup [Docker](https://www.docker.com/products/docker-desktop)
 
-Start the services with Docker Compose
+Start the services (optionally as background processes)
 ```
-docker compose up
+poetry run start [-b]
-```
-or to run in the background
-```
-docker compose up -d
 ```
 
 To stop the services (if running in the background, otherwise just ctrl+c)
 ```
-docker compose down
+poetry run stop
 ```
 
-Check `docker compose help` for more tools available
+MEV container can be attached via
+```
+poetry run attach
+```
+
+Running additional compose commands are possible through standard `docker
+compose ...` calls.  Check `docker compose help` for more tools available
 
 ## Executing scripts
-To run a command, prefix it with
+Inspection is the only simplified api available through poetry at the moment
+with a more generalized api on the horizon.
+
+Inspect scripts must have `-script`, `-block_number` and `-rpc` arguments.
+Using the uniswap inspect from `./examples`
 ```
-docker compose exec mev-inspect <YOUR COMMAND>
+poetry run inspect -script ./examples/uniswap_inspect.py -block_number 11931271 \
+                   -rpc 'http://111.11.11.111:8545'
 ```
 
-For example, to run `testing_file.py`:
+Generalized user defined scripts can still be run through the docker interface as
 ```
 docker compose exec mev-inspect python testing_file.py \
     -block_number 11931271 \
     -rpc 'http://111.11.11.111:8545'
 ```
+### Poetry Scripts
+```bash
+# code check
+poetry run lint # linting via Pylint
+poetry run test # testing and code coverage with Pytest
+poetry run isort # fixing imports 
+poetry run mypy # type checking 
+poetry run black # style guide 
+poetry run pre-commit # runs Black, PyLint and MyPy
+# docker management
+poetry run start [-b] # starts all services, optionally in the background
+poetry run stop # shutsdown all services or just ctrl + c if foreground
+poetry run build # rebuilds containers
+poetry run attach # enters the mev-inspect container in interactive mode
+# launches inspection script
+poetry run inspect -script ... -block_number ... -rpc ...
+```
 
-Or to run the tests:
-```
-docker compose exec mev-inspect python -m unittest tests/*py
-```
 
 ## Rebuilding containers
 After changes to the app's Dockerfile, rebuild with
 ```
-docker compose build
+poetry run build
 ```
 
 ## Using PGAdmin
@@ -63,34 +83,27 @@ docker compose build
     - user / password: see `.env`
 
 ## Contributing
-Contributing requires installing the pre-commit hooks
+Development can be done locally or in the docker container.  Use local if
+contributions can be fully tested without invoking the database related
+services.
 
-1 . Ensure you're using python 3.9
+1. Install dependencies and build python environment
-
-If not, [pyenv](https://github.com/pyenv/pyenv) is a great option for managing python versions
-
-2. Create a virtualenv
 ```
-python3 -m venv venv
+poetry install
+```
+or with docker
+```
+poetry run build
+```
+2. Pre-commit is used to maintain a consistent style, prevent errors and ensure
+   test coverage.  Make sure to fix any errors presented via Black, Pylint and
+   MyPy pre-commit hooks
+```
+poetry run pre-commit
 ```
 
-3. Activate it
+or within docker
-```
-. venv/bin/activate
-```
-(exit with `deactivate`)
-
-4. Install dev libraries
-```
-pip install -r requirements_dev.txt
-```
-
-5. Install pre-commit
-```
-pre-commit install
-```
-
-6. Install pre-commit's dependencies and ensure that it's working
 ```
 pre-commit run --all-files
 ```
+3. Update README if needed

docker-compose.yml

@@ -7,6 +7,7 @@ services:
       - .env
     volumes:
       - .:/app
+    tty: true
 
   db:
     image: postgres:12

pyproject.toml

@@ -0,0 +1,69 @@
+[tool.poetry]
+name = "mev_inspect"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+web3 = "^5.21.0"
+pydantic = "^1.8.2"
+hexbytes = "^0.2.1"
+click = "^8.0.1"
+
+[tool.poetry.dev-dependencies]
+pre-commit = "^2.13.0"
+pylint = "^2.9.5"
+mypy = "^0.910"
+black = "^21.7b0"
+isort = "^5.9.2"
+pytest = "^6.2.4"
+pytest-sugar = "^0.9.4"
+pytest-cov = "^2.12.1"
+coverage = "^5.5"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.scripts]
+lint = 'scripts.dev_tools:lint'
+test = 'scripts.dev_tools:test'
+isort = 'scripts.dev_tools:isort'
+mypy = 'scripts.dev_tools:mypy'
+black = 'scripts.dev_tools:black'
+pre_commit = 'scripts.dev_tools:pre_commit'
+start = 'scripts.docker:start'
+stop = 'scripts.docker:stop'
+build = 'scripts.docker:build'
+attach = 'scripts.docker:attach'
+inspect = 'scripts.docker:inspect'
+
+[tool.black]
+exclude = '''
+/(
+    \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+  | tests/.*/setup.py
+)/
+'''
+
+[tool.isort]
+profile = "black"
+atomic = true
+include_trailing_comma = true
+lines_after_imports = 2
+lines_between_types = 1
+use_parentheses = true
+src_paths = ["poetry", "tests"]
+skip_glob = ["*/setup.py"]
+filter_files = true
+known_first_party = "poetry"

requirements.txt

@@ -1,4 +0,0 @@
-web3==5.20.1
-hexbytes==0.2.1
-argparse==1.4.0
-pydantic==1.8.2

requirements_dev.txt

@@ -1,3 +0,0 @@
-pre-commit==2.13.0
-pylint==2.9.3
-mypy==0.910

run.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-# Source: https://github.com/docker/compose/issues/1926#issuecomment-505294443
-
-# Ah, ha, ha, ha, stayin' alive...
-while :; do :; done & kill -STOP $! && wait $!

scripts/dev_tools.py

@@ -0,0 +1,38 @@
+from subprocess import check_call
+import click
+
+
+def lint():
+    check_call(["pylint", "."])
+
+
+def test():
+    check_call(["pytest", "--cov=mev_inspect", "tests"])
+
+
+@click.command()
+@click.option("-c", required=False, is_flag=True)
+def isort(c: str):
+    """if c is present run isort in diff mode"""
+    if c:
+        check_call(["isort", "."])
+    else:
+        check_call(["isort", "--diff", "."])
+
+
+def mypy():
+    check_call(["mypy", "."])
+
+
+@click.command()
+@click.option("-c", required=False, is_flag=True)
+def black(c: str):
+    """if c is present run black in diff mode"""
+    if c:
+        check_call(["black", "."])
+    else:
+        check_call(["black", "--diff", "--color", "."])
+
+
+def pre_commit():
+    check_call(["pre-commit", "run", "--all-files"])

scripts/docker.py

@@ -0,0 +1,45 @@
+from subprocess import check_call
+import click
+
+
+@click.command()
+@click.option("-b", required=False, is_flag=True)
+def start(b: str):
+    """if d is present background compose"""
+    if b:
+        check_call(["docker", "compose", "up", "-d"])
+        click.echo("docker running in the background...")
+    else:
+        check_call(["docker", "compose", "up"])
+
+
+def stop():
+    check_call(["docker", "compose", "down"])
+
+
+def build():
+    check_call(["docker", "compose", "build"])
+
+
+def attach():
+    check_call(["docker", "exec", "-it", "mev-inspect-py_mev-inspect_1", "bash"])
+
+
+@click.command()
+@click.option("-script", help="inspect script", default="./examples/uniswap_inspect.py")
+@click.option("-block_num", help="block number to inspect", default=11931271)
+@click.option("-rpc", help="rpc address", default="http://111.11.11.111:8545")
+def inspect(script: str, block_num: int, rpc: str):
+    """Runs mev-inspect scripts through docker services"""
+    check_call(
+        [
+            "docker",
+            "compose",
+            "exec",
+            "mev-inspect",
+            "python",
+            script,
+            f"-block_number {block_num}",
+            f"-rpc {rpc}",
+        ]
+    )