From 053c29cf209876a3f7a7a81d81365cd205939411 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Wed, 17 Nov 2021 21:59:12 -0500 Subject: [PATCH 01/26] Add placeholder for price commands --- cli.py | 6 ++++++ mev | 16 ++++++++++++++++ pyproject.toml | 1 + 3 files changed, 23 insertions(+) diff --git a/cli.py b/cli.py index 12d62c7..b1c0acf 100644 --- a/cli.py +++ b/cli.py @@ -79,6 +79,12 @@ async def inspect_many_blocks_command( ) +@cli.command() +@coro +async def fetch_all_prices(): + print("fetching") + + def get_rpc_url() -> str: return os.environ["RPC_URL"] diff --git a/mev b/mev index 7d7f79f..69e29c4 100755 --- a/mev +++ b/mev @@ -56,6 +56,22 @@ case "$1" in echo "Fetching block $block_number" kubectl exec -ti deploy/mev-inspect -- poetry run fetch-block $block_number ;; + prices) + shift + case "$1" in + fetch-latest) + echo "running fetch-latest" + ;; + fetch-all) + echo "running fetch-all" + kubectl exec -ti deploy/mev-inspect -- \ + poetry run fetch-all-prices + ;; + *) + echo "prices usage: "$1" {fetch-latest|fetch-all}" + exit 1 + esac + ;; exec) shift kubectl exec -ti deploy/mev-inspect -- $@ diff --git a/pyproject.toml b/pyproject.toml index 9217449..5d69c1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ build-backend = "poetry.core.masonry.api" inspect-block = 'cli:inspect_block_command' inspect-many-blocks = 'cli:inspect_many_blocks_command' fetch-block = 'cli:fetch_block_command' +fetch-all-prices = 'cli:fetch_all_prices' [tool.black] exclude = ''' From 2dc14218bfc102a2397b2f0326f4c6f881c36c7d Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Wed, 17 Nov 2021 23:07:25 -0500 Subject: [PATCH 02/26] Add support for fetching all supported prices --- cli.py | 3 +++ mev_inspect/coinbase.py | 25 +++++++++++++++++++++++++ mev_inspect/prices.py | 29 +++++++++++++++++++++++++++++ mev_inspect/schemas/coinbase.py | 20 ++++++++++++++++++++ mev_inspect/schemas/prices.py | 7 +++++++ mev_inspect/schemas/utils.py | 2 +- 6 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 mev_inspect/coinbase.py create mode 100644 mev_inspect/prices.py create mode 100644 mev_inspect/schemas/coinbase.py create mode 100644 mev_inspect/schemas/prices.py diff --git a/cli.py b/cli.py index b1c0acf..5e04f6a 100644 --- a/cli.py +++ b/cli.py @@ -7,6 +7,7 @@ import click from mev_inspect.concurrency import coro from mev_inspect.db import get_inspect_session, get_trace_session from mev_inspect.inspector import MEVInspector +from mev_inspect.prices import fetch_all_supported_prices RPC_URL_ENV = "RPC_URL" @@ -83,6 +84,8 @@ async def inspect_many_blocks_command( @coro async def fetch_all_prices(): print("fetching") + prices = await fetch_all_supported_prices() + print(prices[0]) def get_rpc_url() -> str: diff --git a/mev_inspect/coinbase.py b/mev_inspect/coinbase.py new file mode 100644 index 0000000..857288b --- /dev/null +++ b/mev_inspect/coinbase.py @@ -0,0 +1,25 @@ +import aiohttp + +from mev_inspect.classifiers.specs.weth import WETH_ADDRESS +from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS +from mev_inspect.schemas.coinbase import CoinbasePrices, CoinbasePricesResponse + + +COINBASE_API_BASE = "https://www.coinbase.com/api/v2" +COINBASE_TOKEN_NAMES = { + WETH_ADDRESS: "weth", + ETH_TOKEN_ADDRESS: "ethereum", +} + + +async def fetch_coinbase_prices(token_address: str) -> CoinbasePrices: + if token_address not in COINBASE_TOKEN_NAMES: + raise ValueError(f"Unsupported token_address {token_address}") + + coinbase_token_name = COINBASE_TOKEN_NAMES[token_address] + url = f"{COINBASE_API_BASE}/assets/prices/{coinbase_token_name}" + + async with aiohttp.ClientSession() as session: + async with session.get(url, params={"base": "USD"}) as response: + json_data = await response.json() + return CoinbasePricesResponse(**json_data).data.prices diff --git a/mev_inspect/prices.py b/mev_inspect/prices.py new file mode 100644 index 0000000..8f0ac4a --- /dev/null +++ b/mev_inspect/prices.py @@ -0,0 +1,29 @@ +from typing import List + +from mev_inspect.classifiers.specs.weth import WETH_ADDRESS +from mev_inspect.coinbase import fetch_coinbase_prices +from mev_inspect.schemas.prices import Price +from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS + + +SUPPORTED_TOKENS = [ + WETH_ADDRESS, + ETH_TOKEN_ADDRESS, +] + + +async def fetch_all_supported_prices() -> List[Price]: + prices = [] + + for token_address in SUPPORTED_TOKENS: + coinbase_prices = await fetch_coinbase_prices(token_address) + for usd_price, timestamp_seconds in coinbase_prices.all.prices: + price = Price( + token_address=token_address, + usd_price=usd_price, + timestamp_seconds=timestamp_seconds, + ) + + prices.append(price) + + return prices diff --git a/mev_inspect/schemas/coinbase.py b/mev_inspect/schemas/coinbase.py new file mode 100644 index 0000000..fca7bab --- /dev/null +++ b/mev_inspect/schemas/coinbase.py @@ -0,0 +1,20 @@ +from typing import List, Tuple + +from pydantic import BaseModel + + +class CoinbasePricesEntry(BaseModel): + # tuple of price and timestamp + prices: List[Tuple[float, int]] + + +class CoinbasePrices(BaseModel): + all: CoinbasePricesEntry + + +class CoinbasePricesDataResponse(BaseModel): + prices: CoinbasePrices + + +class CoinbasePricesResponse(BaseModel): + data: CoinbasePricesDataResponse diff --git a/mev_inspect/schemas/prices.py b/mev_inspect/schemas/prices.py new file mode 100644 index 0000000..55abe0f --- /dev/null +++ b/mev_inspect/schemas/prices.py @@ -0,0 +1,7 @@ +from pydantic import BaseModel + + +class Price(BaseModel): + token_address: str + timestamp_seconds: int + usd_price: float diff --git a/mev_inspect/schemas/utils.py b/mev_inspect/schemas/utils.py index e3b53f6..1d15876 100644 --- a/mev_inspect/schemas/utils.py +++ b/mev_inspect/schemas/utils.py @@ -1,8 +1,8 @@ import json from hexbytes import HexBytes -from web3.datastructures import AttributeDict from pydantic import BaseModel +from web3.datastructures import AttributeDict def to_camel(string: str) -> str: From bed8520bc88728fa829c4e6cc6c83be8dc30b9dd Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 18 Nov 2021 11:55:42 -0500 Subject: [PATCH 03/26] Write prices on fetch-all --- cli.py | 9 +++++++-- mev_inspect/prices.py | 2 +- mev_inspect/schemas/prices.py | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cli.py b/cli.py index 5e04f6a..fd3a30e 100644 --- a/cli.py +++ b/cli.py @@ -5,6 +5,7 @@ import sys import click from mev_inspect.concurrency import coro +from mev_inspect.crud.prices import write_prices from mev_inspect.db import get_inspect_session, get_trace_session from mev_inspect.inspector import MEVInspector from mev_inspect.prices import fetch_all_supported_prices @@ -83,9 +84,13 @@ async def inspect_many_blocks_command( @cli.command() @coro async def fetch_all_prices(): - print("fetching") + inspect_db_session = get_inspect_session() + + print("Fetching prices") prices = await fetch_all_supported_prices() - print(prices[0]) + + print("Writing prices") + write_prices(inspect_db_session, prices) def get_rpc_url() -> str: diff --git a/mev_inspect/prices.py b/mev_inspect/prices.py index 8f0ac4a..8abe23d 100644 --- a/mev_inspect/prices.py +++ b/mev_inspect/prices.py @@ -21,7 +21,7 @@ async def fetch_all_supported_prices() -> List[Price]: price = Price( token_address=token_address, usd_price=usd_price, - timestamp_seconds=timestamp_seconds, + timestamp=timestamp_seconds, ) prices.append(price) diff --git a/mev_inspect/schemas/prices.py b/mev_inspect/schemas/prices.py index 55abe0f..40e5c48 100644 --- a/mev_inspect/schemas/prices.py +++ b/mev_inspect/schemas/prices.py @@ -1,7 +1,9 @@ +from datetime import datetime + from pydantic import BaseModel class Price(BaseModel): token_address: str - timestamp_seconds: int + timestamp: datetime usd_price: float From 5b59427d4f8b34f69a42394016391a71d2180f85 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 18 Nov 2021 13:43:21 -0500 Subject: [PATCH 04/26] Write prices. Ignore duplicates --- mev_inspect/crud/prices.py | 17 +++++++++++++++++ mev_inspect/models/prices.py | 11 +++++++++++ 2 files changed, 28 insertions(+) create mode 100644 mev_inspect/crud/prices.py create mode 100644 mev_inspect/models/prices.py diff --git a/mev_inspect/crud/prices.py b/mev_inspect/crud/prices.py new file mode 100644 index 0000000..97f4166 --- /dev/null +++ b/mev_inspect/crud/prices.py @@ -0,0 +1,17 @@ +from typing import List + +from sqlalchemy.dialects.postgresql import insert + +from mev_inspect.models.prices import PriceModel +from mev_inspect.schemas.prices import Price + + +def write_prices(db_session, prices: List[Price]) -> None: + insert_statement = ( + insert(PriceModel.__table__) + .values([price.dict() for price in prices]) + .on_conflict_do_nothing() + ) + + db_session.execute(insert_statement) + db_session.commit() diff --git a/mev_inspect/models/prices.py b/mev_inspect/models/prices.py new file mode 100644 index 0000000..cb41c83 --- /dev/null +++ b/mev_inspect/models/prices.py @@ -0,0 +1,11 @@ +from sqlalchemy import Column, Numeric, String, TIMESTAMP + +from .base import Base + + +class PriceModel(Base): + __tablename__ = "usd_prices" + + timestamp = Column(TIMESTAMP, nullable=False, primary_key=True) + usd_price = Column(Numeric, nullable=False) + token_address = Column(String, nullable=False, primary_key=True) From d499983f32687a2d73c2f5880f2efd75b505fc70 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 18 Nov 2021 13:45:25 -0500 Subject: [PATCH 05/26] Remove fetch-latest for now --- mev | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mev b/mev index 69e29c4..5893782 100755 --- a/mev +++ b/mev @@ -59,16 +59,13 @@ case "$1" in prices) shift case "$1" in - fetch-latest) - echo "running fetch-latest" - ;; fetch-all) echo "running fetch-all" kubectl exec -ti deploy/mev-inspect -- \ poetry run fetch-all-prices ;; *) - echo "prices usage: "$1" {fetch-latest|fetch-all}" + echo "prices usage: "$1" {fetch-all}" exit 1 esac ;; From 023205c25b60cdc2b0929952f14e181431d96b23 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 18 Nov 2021 13:47:59 -0500 Subject: [PATCH 06/26] Print => logger --- cli.py | 5 +++-- mev | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cli.py b/cli.py index fd3a30e..2a78b75 100644 --- a/cli.py +++ b/cli.py @@ -13,6 +13,7 @@ from mev_inspect.prices import fetch_all_supported_prices RPC_URL_ENV = "RPC_URL" logging.basicConfig(stream=sys.stdout, level=logging.INFO) +logger = logging.getLogger(__name__) @click.group() @@ -86,10 +87,10 @@ async def inspect_many_blocks_command( async def fetch_all_prices(): inspect_db_session = get_inspect_session() - print("Fetching prices") + logger.info("Fetching prices") prices = await fetch_all_supported_prices() - print("Writing prices") + logger.info("Writing prices") write_prices(inspect_db_session, prices) diff --git a/mev b/mev index 5893782..e326f77 100755 --- a/mev +++ b/mev @@ -60,7 +60,7 @@ case "$1" in shift case "$1" in fetch-all) - echo "running fetch-all" + echo "Running price fetch-all" kubectl exec -ti deploy/mev-inspect -- \ poetry run fetch-all-prices ;; From f5233a17fd2e6c6abb7f0cd3104354435996ed21 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 18 Nov 2021 13:56:07 -0500 Subject: [PATCH 07/26] Rename to prices table --- mev_inspect/models/prices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mev_inspect/models/prices.py b/mev_inspect/models/prices.py index cb41c83..86bf3e0 100644 --- a/mev_inspect/models/prices.py +++ b/mev_inspect/models/prices.py @@ -4,7 +4,7 @@ from .base import Base class PriceModel(Base): - __tablename__ = "usd_prices" + __tablename__ = "prices" timestamp = Column(TIMESTAMP, nullable=False, primary_key=True) usd_price = Column(Numeric, nullable=False) From 4f34316afb7554082822dbe54dda7ff98d8610f1 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Fri, 26 Nov 2021 21:03:57 -0500 Subject: [PATCH 08/26] COINBASE_TOKEN_NAMES => COINBASE_TOKEN_NAME_BY_ADDRESS --- mev_inspect/coinbase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mev_inspect/coinbase.py b/mev_inspect/coinbase.py index 857288b..215a379 100644 --- a/mev_inspect/coinbase.py +++ b/mev_inspect/coinbase.py @@ -6,17 +6,17 @@ from mev_inspect.schemas.coinbase import CoinbasePrices, CoinbasePricesResponse COINBASE_API_BASE = "https://www.coinbase.com/api/v2" -COINBASE_TOKEN_NAMES = { +COINBASE_TOKEN_NAME_BY_ADDRESS = { WETH_ADDRESS: "weth", ETH_TOKEN_ADDRESS: "ethereum", } async def fetch_coinbase_prices(token_address: str) -> CoinbasePrices: - if token_address not in COINBASE_TOKEN_NAMES: + if token_address not in COINBASE_TOKEN_NAME_BY_ADDRESS: raise ValueError(f"Unsupported token_address {token_address}") - coinbase_token_name = COINBASE_TOKEN_NAMES[token_address] + coinbase_token_name = COINBASE_TOKEN_NAME_BY_ADDRESS[token_address] url = f"{COINBASE_API_BASE}/assets/prices/{coinbase_token_name}" async with aiohttp.ClientSession() as session: From a8c1728e357032fd2bdf238a4858ed469c67ef55 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Fri, 19 Nov 2021 08:20:40 -0500 Subject: [PATCH 09/26] Save progress --- Dockerfile | 2 +- Tiltfile | 6 +- k8s/mev-inspect-backfill/templates/job.yaml | 3 +- k8s/mev-inspect-prices/.helmignore | 23 +++++++ k8s/mev-inspect-prices/Chart.yaml | 24 +++++++ k8s/mev-inspect-prices/templates/_helpers.tpl | 62 +++++++++++++++++++ k8s/mev-inspect-prices/templates/cronjob.yaml | 36 +++++++++++ k8s/mev-inspect-prices/values.yaml | 7 +++ k8s/mev-inspect/templates/deployment.yaml | 1 + 9 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 k8s/mev-inspect-prices/.helmignore create mode 100644 k8s/mev-inspect-prices/Chart.yaml create mode 100644 k8s/mev-inspect-prices/templates/_helpers.tpl create mode 100644 k8s/mev-inspect-prices/templates/cronjob.yaml create mode 100644 k8s/mev-inspect-prices/values.yaml diff --git a/Dockerfile b/Dockerfile index 5801d11..5a950e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,4 @@ COPY . /app # easter eggs 😝 RUN echo "PS1='🕵️:\[\033[1;36m\]\h \[\033[1;34m\]\W\[\033[0;35m\]\[\033[1;36m\]$ \[\033[0m\]'" >> ~/.bashrc -ENTRYPOINT [ "/app/entrypoint.sh"] +ENTRYPOINT [ "poetry"] diff --git a/Tiltfile b/Tiltfile index 9f8ec7d..585d18f 100644 --- a/Tiltfile +++ b/Tiltfile @@ -31,16 +31,20 @@ k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = { # })) docker_build_with_restart("mev-inspect-py", ".", - entrypoint="/app/entrypoint.sh", + entrypoint="poetry", live_update=[ sync(".", "/app"), run("cd /app && poetry install", trigger="./pyproject.toml"), ], + platform='linux/arm64', ) k8s_yaml(helm('./k8s/mev-inspect', name='mev-inspect')) k8s_resource(workload="mev-inspect", resource_deps=["postgresql-postgresql"]) +k8s_yaml(helm('./k8s/mev-inspect-prices', name='mev-inspect-prices')) +k8s_resource(workload="mev-inspect-prices", resource_deps=["postgresql-postgresql"]) + local_resource( 'pg-port-forward', serve_cmd='kubectl port-forward --namespace default svc/postgresql 5432:5432', diff --git a/k8s/mev-inspect-backfill/templates/job.yaml b/k8s/mev-inspect-backfill/templates/job.yaml index 35f07b0..6f0bc0b 100644 --- a/k8s/mev-inspect-backfill/templates/job.yaml +++ b/k8s/mev-inspect-backfill/templates/job.yaml @@ -21,8 +21,7 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - command: - - poetry + args: - run - inspect-many-blocks - {{ .Values.command.startBlockNumber | quote }} diff --git a/k8s/mev-inspect-prices/.helmignore b/k8s/mev-inspect-prices/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/k8s/mev-inspect-prices/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s/mev-inspect-prices/Chart.yaml b/k8s/mev-inspect-prices/Chart.yaml new file mode 100644 index 0000000..5083c60 --- /dev/null +++ b/k8s/mev-inspect-prices/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: mev-inspect-prices +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/k8s/mev-inspect-prices/templates/_helpers.tpl b/k8s/mev-inspect-prices/templates/_helpers.tpl new file mode 100644 index 0000000..1d8a659 --- /dev/null +++ b/k8s/mev-inspect-prices/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "mev-inspect-prices.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "mev-inspect-prices.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "mev-inspect-prices.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "mev-inspect-prices.labels" -}} +helm.sh/chart: {{ include "mev-inspect-prices.chart" . }} +{{ include "mev-inspect-prices.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "mev-inspect-prices.selectorLabels" -}} +app.kubernetes.io/name: {{ include "mev-inspect-prices.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "mev-inspect-prices.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "mev-inspect-prices.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/k8s/mev-inspect-prices/templates/cronjob.yaml b/k8s/mev-inspect-prices/templates/cronjob.yaml new file mode 100644 index 0000000..4c982bc --- /dev/null +++ b/k8s/mev-inspect-prices/templates/cronjob.yaml @@ -0,0 +1,36 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "mev-inspect-prices.fullname" . }} +spec: + completions: 1 + ttlSecondsAfterFinished: 5 + schedule: "*/1 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - run + - fetch-all-prices + env: + - name: POSTGRES_HOST + valueFrom: + secretKeyRef: + name: mev-inspect-db-credentials + key: host + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: mev-inspect-db-credentials + key: username + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: mev-inspect-db-credentials + key: password + restartPolicy: OnFailure diff --git a/k8s/mev-inspect-prices/values.yaml b/k8s/mev-inspect-prices/values.yaml new file mode 100644 index 0000000..77238c0 --- /dev/null +++ b/k8s/mev-inspect-prices/values.yaml @@ -0,0 +1,7 @@ +image: + repository: mev-inspect-py + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" diff --git a/k8s/mev-inspect/templates/deployment.yaml b/k8s/mev-inspect/templates/deployment.yaml index 9aa4a50..ec2045c 100644 --- a/k8s/mev-inspect/templates/deployment.yaml +++ b/k8s/mev-inspect/templates/deployment.yaml @@ -30,6 +30,7 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + args: ["run", "python", "loop.py"] livenessProbe: exec: command: From 34aca861ccfb37a81801aa577b73a54be2d048d6 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 22 Nov 2021 13:03:50 -0500 Subject: [PATCH 10/26] Use poetry directly instead of entrypoint script --- Dockerfile | 3 ++- Tiltfile | 4 +--- entrypoint.sh | 3 --- 3 files changed, 3 insertions(+), 7 deletions(-) delete mode 100755 entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 5a950e8..569df05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,5 @@ COPY . /app # easter eggs 😝 RUN echo "PS1='🕵️:\[\033[1;36m\]\h \[\033[1;34m\]\W\[\033[0;35m\]\[\033[1;36m\]$ \[\033[0m\]'" >> ~/.bashrc -ENTRYPOINT [ "poetry"] +ENTRYPOINT [ "poetry" ] +CMD [ "run", "python", "loop.py" ] diff --git a/Tiltfile b/Tiltfile index 585d18f..a828272 100644 --- a/Tiltfile +++ b/Tiltfile @@ -30,14 +30,12 @@ k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = { # "host": "trace-db-postgresql", # })) -docker_build_with_restart("mev-inspect-py", ".", - entrypoint="poetry", +docker_build("mev-inspect-py", ".", live_update=[ sync(".", "/app"), run("cd /app && poetry install", trigger="./pyproject.toml"), ], - platform='linux/arm64', ) k8s_yaml(helm('./k8s/mev-inspect', name='mev-inspect')) k8s_resource(workload="mev-inspect", resource_deps=["postgresql-postgresql"]) diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100755 index 54f1f8a..0000000 --- a/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -python loop.py From c83577b04c4b4acb6493e03513681b2e29967018 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 22 Nov 2021 13:06:30 -0500 Subject: [PATCH 11/26] Remove restart --- Tiltfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Tiltfile b/Tiltfile index a828272..1be021c 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,5 +1,4 @@ load("ext://helm_remote", "helm_remote") -load("ext://restart_process", "docker_build_with_restart") load("ext://secret", "secret_from_dict") load("ext://configmap", "configmap_from_dict") From 391314b9d6ee618dfba8e9125a4b510c828aca2c Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 22 Nov 2021 13:51:35 -0500 Subject: [PATCH 12/26] Limit successful history instead of ttl --- k8s/mev-inspect-prices/templates/cronjob.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/k8s/mev-inspect-prices/templates/cronjob.yaml b/k8s/mev-inspect-prices/templates/cronjob.yaml index 4c982bc..1c1379b 100644 --- a/k8s/mev-inspect-prices/templates/cronjob.yaml +++ b/k8s/mev-inspect-prices/templates/cronjob.yaml @@ -3,9 +3,8 @@ kind: CronJob metadata: name: {{ include "mev-inspect-prices.fullname" . }} spec: - completions: 1 - ttlSecondsAfterFinished: 5 schedule: "*/1 * * * *" + successfulJobsHistoryLimit: 0 jobTemplate: spec: template: @@ -33,4 +32,4 @@ spec: secretKeyRef: name: mev-inspect-db-credentials key: password - restartPolicy: OnFailure + restartPolicy: Never From 9a076a6b4cf576dd5c619e45423d99aa3b4d4601 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 22 Nov 2021 13:52:27 -0500 Subject: [PATCH 13/26] Don't run prices by default --- Tiltfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tiltfile b/Tiltfile index 1be021c..afc0d92 100644 --- a/Tiltfile +++ b/Tiltfile @@ -39,8 +39,9 @@ docker_build("mev-inspect-py", ".", k8s_yaml(helm('./k8s/mev-inspect', name='mev-inspect')) k8s_resource(workload="mev-inspect", resource_deps=["postgresql-postgresql"]) -k8s_yaml(helm('./k8s/mev-inspect-prices', name='mev-inspect-prices')) -k8s_resource(workload="mev-inspect-prices", resource_deps=["postgresql-postgresql"]) +# uncomment to enable price monitor +# k8s_yaml(helm('./k8s/mev-inspect-prices', name='mev-inspect-prices')) +# k8s_resource(workload="mev-inspect-prices", resource_deps=["postgresql-postgresql"]) local_resource( 'pg-port-forward', From 018fb8c73bdfe2745a6e4caae58c0e64734397ed Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 22 Nov 2021 13:54:12 -0500 Subject: [PATCH 14/26] Run hourly --- k8s/mev-inspect-prices/templates/cronjob.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/mev-inspect-prices/templates/cronjob.yaml b/k8s/mev-inspect-prices/templates/cronjob.yaml index 1c1379b..900d0ab 100644 --- a/k8s/mev-inspect-prices/templates/cronjob.yaml +++ b/k8s/mev-inspect-prices/templates/cronjob.yaml @@ -3,7 +3,7 @@ kind: CronJob metadata: name: {{ include "mev-inspect-prices.fullname" . }} spec: - schedule: "*/1 * * * *" + schedule: "0 */1 * * *" successfulJobsHistoryLimit: 0 jobTemplate: spec: From c619c208781f0738c7828d750c0bc9277b619a42 Mon Sep 17 00:00:00 2001 From: Robert Miller Date: Fri, 26 Nov 2021 21:29:53 -0500 Subject: [PATCH 15/26] bug: add a missing parentheses --- mev_inspect/inspect_block.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py index c56e024..361f28a 100644 --- a/mev_inspect/inspect_block.py +++ b/mev_inspect/inspect_block.py @@ -17,7 +17,7 @@ from mev_inspect.crud.punks import ( write_punk_snipes, delete_punk_bids_for_block, write_punk_bids, - +) from mev_inspect.crud.blocks import ( delete_block, write_block, From c1ba63ef8154783757687946acc8c3ba3cb1084a Mon Sep 17 00:00:00 2001 From: Robert Miller Date: Fri, 26 Nov 2021 21:34:16 -0500 Subject: [PATCH 16/26] style: formatting --- mev_inspect/inspect_block.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py index 361f28a..0014c03 100644 --- a/mev_inspect/inspect_block.py +++ b/mev_inspect/inspect_block.py @@ -21,7 +21,6 @@ from mev_inspect.crud.punks import ( from mev_inspect.crud.blocks import ( delete_block, write_block, - ) from mev_inspect.crud.traces import ( delete_classified_traces_for_block, From cd01298ba659b07324da26004a610a4b7940db81 Mon Sep 17 00:00:00 2001 From: Shea Ketsdever Date: Sun, 28 Nov 2021 14:51:24 -0800 Subject: [PATCH 17/26] Bancor classifier --- mev_inspect/abis/bancor/BancorNetwork.json | 1 + mev_inspect/classifiers/helpers.py | 37 +++++++++++++ mev_inspect/classifiers/specs/__init__.py | 2 + mev_inspect/classifiers/specs/bancor.py | 48 +++++++++++++++++ mev_inspect/schemas/traces.py | 1 + tests/test_swaps.py | 62 ++++++++++++++++++++-- 6 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 mev_inspect/abis/bancor/BancorNetwork.json create mode 100644 mev_inspect/classifiers/specs/bancor.py diff --git a/mev_inspect/abis/bancor/BancorNetwork.json b/mev_inspect/abis/bancor/BancorNetwork.json new file mode 100644 index 0000000..99a7e83 --- /dev/null +++ b/mev_inspect/abis/bancor/BancorNetwork.json @@ -0,0 +1 @@ +[{"constant":false,"inputs":[{"name":"_onlyOwnerCanUpdateRegistry","type":"bool"}],"name":"restrictRegistryUpdate","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_token","type":"address"},{"name":"_register","type":"bool"}],"name":"registerEtherToken","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"}],"name":"getReturnByPath","outputs":[{"name":"","type":"uint256"},{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"},{"name":"_minReturn","type":"uint256"},{"name":"_beneficiary","type":"address"},{"name":"_affiliateAccount","type":"address"},{"name":"_affiliateFee","type":"uint256"}],"name":"claimAndConvertFor2","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"onlyOwnerCanUpdateRegistry","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"updateRegistry","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"},{"name":"_minReturn","type":"uint256"},{"name":"_affiliateAccount","type":"address"},{"name":"_affiliateFee","type":"uint256"}],"name":"convert2","outputs":[{"name":"","type":"uint256"}],"payable":true,"stateMutability":"payable","type":"function"},{"constant":true,"inputs":[],"name":"maxAffiliateFee","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_token","type":"address"},{"name":"_to","type":"address"},{"name":"_amount","type":"uint256"}],"name":"withdrawTokens","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"prevRegistry","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[],"name":"acceptOwnership","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"registry","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"}],"name":"rateByPath","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"address"}],"name":"etherTokens","outputs":[{"name":"","type":"bool"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_path","type":"address[]"},{"name":"_bancorX","type":"address"},{"name":"_conversionId","type":"uint256"},{"name":"_minReturn","type":"uint256"},{"name":"_beneficiary","type":"address"}],"name":"completeXConversion","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"owner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"},{"name":"_minReturn","type":"uint256"},{"name":"_beneficiary","type":"address"},{"name":"_affiliateAccount","type":"address"},{"name":"_affiliateFee","type":"uint256"}],"name":"convertFor2","outputs":[{"name":"","type":"uint256"}],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"},{"name":"_minReturn","type":"uint256"},{"name":"_beneficiary","type":"address"}],"name":"claimAndConvertFor","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[],"name":"restoreRegistry","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"},{"name":"_minReturn","type":"uint256"},{"name":"_beneficiary","type":"address"},{"name":"_affiliateAccount","type":"address"},{"name":"_affiliateFee","type":"uint256"}],"name":"convertByPath","outputs":[{"name":"","type":"uint256"}],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"},{"name":"_minReturn","type":"uint256"},{"name":"_targetBlockchain","type":"bytes32"},{"name":"_targetAccount","type":"bytes32"},{"name":"_conversionId","type":"uint256"}],"name":"xConvert","outputs":[{"name":"","type":"uint256"}],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"},{"name":"_minReturn","type":"uint256"}],"name":"claimAndConvert","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"},{"name":"_minReturn","type":"uint256"},{"name":"_beneficiary","type":"address"}],"name":"convertFor","outputs":[{"name":"","type":"uint256"}],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"},{"name":"_minReturn","type":"uint256"},{"name":"_targetBlockchain","type":"bytes32"},{"name":"_targetAccount","type":"bytes32"},{"name":"_conversionId","type":"uint256"},{"name":"_affiliateAccount","type":"address"},{"name":"_affiliateFee","type":"uint256"}],"name":"xConvert2","outputs":[{"name":"","type":"uint256"}],"payable":true,"stateMutability":"payable","type":"function"},{"constant":true,"inputs":[],"name":"newOwner","outputs":[{"name":"","type":"address"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"_sourceToken","type":"address"},{"name":"_targetToken","type":"address"}],"name":"conversionPath","outputs":[{"name":"","type":"address[]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":false,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"},{"name":"_minReturn","type":"uint256"},{"name":"_affiliateAccount","type":"address"},{"name":"_affiliateFee","type":"uint256"}],"name":"claimAndConvert2","outputs":[{"name":"","type":"uint256"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_newOwner","type":"address"}],"name":"transferOwnership","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"_path","type":"address[]"},{"name":"_amount","type":"uint256"},{"name":"_minReturn","type":"uint256"}],"name":"convert","outputs":[{"name":"","type":"uint256"}],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[{"name":"_maxAffiliateFee","type":"uint256"}],"name":"setMaxAffiliateFee","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"inputs":[{"name":"_registry","type":"address"}],"payable":false,"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"name":"_smartToken","type":"address"},{"indexed":true,"name":"_fromToken","type":"address"},{"indexed":true,"name":"_toToken","type":"address"},{"indexed":false,"name":"_fromAmount","type":"uint256"},{"indexed":false,"name":"_toAmount","type":"uint256"},{"indexed":false,"name":"_trader","type":"address"}],"name":"Conversion","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"_prevOwner","type":"address"},{"indexed":true,"name":"_newOwner","type":"address"}],"name":"OwnerUpdate","type":"event"}] \ No newline at end of file diff --git a/mev_inspect/classifiers/helpers.py b/mev_inspect/classifiers/helpers.py index 58d6440..f120005 100644 --- a/mev_inspect/classifiers/helpers.py +++ b/mev_inspect/classifiers/helpers.py @@ -55,6 +55,43 @@ def create_swap_from_transfers( ) +def create_swap_from_transfers_not_including_pool( + trace: DecodedCallTrace, + pool_address: str, + recipient_address: str, + prior_transfers: List[Transfer], + child_transfers: List[Transfer], +) -> Optional[Swap]: + transfers_from_recipient = _filter_transfers( + [*prior_transfers, *child_transfers], from_address=recipient_address + ) + transfers_to_recipient = _filter_transfers( + child_transfers, to_address=recipient_address + ) + + if len(transfers_from_recipient) != 1 or len(transfers_to_recipient) != 1: + return None + + transfer_in = transfers_from_recipient[0] + transfer_out = transfers_to_recipient[0] + + return Swap( + abi_name=trace.abi_name, + transaction_hash=trace.transaction_hash, + block_number=trace.block_number, + trace_address=trace.trace_address, + contract_address=pool_address, + protocol=trace.protocol, + from_address=transfer_in.from_address, + to_address=transfer_out.to_address, + token_in_address=transfer_in.token_address, + token_in_amount=transfer_in.amount, + token_out_address=transfer_out.token_address, + token_out_amount=transfer_out.amount, + error=trace.error, + ) + + def _build_eth_transfer(trace: ClassifiedTrace) -> Transfer: return Transfer( block_number=trace.block_number, diff --git a/mev_inspect/classifiers/specs/__init__.py b/mev_inspect/classifiers/specs/__init__.py index 6874e01..aa5b28a 100644 --- a/mev_inspect/classifiers/specs/__init__.py +++ b/mev_inspect/classifiers/specs/__init__.py @@ -12,6 +12,7 @@ from .zero_ex import ZEROX_CLASSIFIER_SPECS from .balancer import BALANCER_CLASSIFIER_SPECS from .compound import COMPOUND_CLASSIFIER_SPECS from .cryptopunks import CRYPTOPUNKS_CLASSIFIER_SPECS +from .bancor import BANCOR_CLASSIFIER_SPECS ALL_CLASSIFIER_SPECS = ( ERC20_CLASSIFIER_SPECS @@ -23,6 +24,7 @@ ALL_CLASSIFIER_SPECS = ( + BALANCER_CLASSIFIER_SPECS + COMPOUND_CLASSIFIER_SPECS + CRYPTOPUNKS_CLASSIFIER_SPECS + + BANCOR_CLASSIFIER_SPECS ) _SPECS_BY_ABI_NAME_AND_PROTOCOL: Dict[ diff --git a/mev_inspect/classifiers/specs/bancor.py b/mev_inspect/classifiers/specs/bancor.py new file mode 100644 index 0000000..e8b15f6 --- /dev/null +++ b/mev_inspect/classifiers/specs/bancor.py @@ -0,0 +1,48 @@ +from typing import Optional, List +from mev_inspect.schemas.transfers import Transfer +from mev_inspect.schemas.swaps import Swap +from mev_inspect.schemas.traces import ( + DecodedCallTrace, + Protocol, +) +from mev_inspect.schemas.classifiers import ( + ClassifierSpec, + SwapClassifier, +) +from mev_inspect.classifiers.helpers import ( + create_swap_from_transfers_not_including_pool, +) + +BANCOR_NETWORK_ABI_NAME = "BancorNetwork" +BANCOR_NETWORK_CONTRACT_ADDRESS = "0x2F9EC37d6CcFFf1caB21733BdaDEdE11c823cCB0" + + +class BancorSwapClassifier(SwapClassifier): + @staticmethod + def parse_swap( + trace: DecodedCallTrace, + prior_transfers: List[Transfer], + child_transfers: List[Transfer], + ) -> Optional[Swap]: + recipient_address = trace.from_address + + swap = create_swap_from_transfers_not_including_pool( + trace, + BANCOR_NETWORK_CONTRACT_ADDRESS, + recipient_address, + prior_transfers, + child_transfers, + ) + return swap + + +BANCOR_NETWORK_SPEC = ClassifierSpec( + abi_name=BANCOR_NETWORK_ABI_NAME, + protocol=Protocol.bancor, + classifiers={ + "convertByPath(address[],uint256,uint256,address,address,uint256)": BancorSwapClassifier, + }, + valid_contract_addresses=[BANCOR_NETWORK_CONTRACT_ADDRESS], +) + +BANCOR_CLASSIFIER_SPECS = [BANCOR_NETWORK_SPEC] diff --git a/mev_inspect/schemas/traces.py b/mev_inspect/schemas/traces.py index 6b4480b..6ea5862 100644 --- a/mev_inspect/schemas/traces.py +++ b/mev_inspect/schemas/traces.py @@ -47,6 +47,7 @@ class Protocol(Enum): compound_v2 = "compound_v2" cream = "cream" cryptopunks = "cryptopunks" + bancor = "bancor" class ClassifiedTrace(Trace): diff --git a/tests/test_swaps.py b/tests/test_swaps.py index 68e9071..e51c9b6 100644 --- a/tests/test_swaps.py +++ b/tests/test_swaps.py @@ -4,6 +4,10 @@ from mev_inspect.classifiers.specs.uniswap import ( UNISWAP_V2_PAIR_ABI_NAME, UNISWAP_V3_POOL_ABI_NAME, ) +from mev_inspect.classifiers.specs.bancor import ( + BANCOR_NETWORK_ABI_NAME, + BANCOR_NETWORK_CONTRACT_ADDRESS, +) from mev_inspect.schemas.traces import Protocol from .helpers import ( @@ -23,12 +27,14 @@ def test_swaps( first_transaction_hash, second_transaction_hash, third_transaction_hash, - ] = get_transaction_hashes(3) + fourth_transaction_hash, + ] = get_transaction_hashes(4) [ alice_address, bob_address, carl_address, + danielle_address, first_token_in_address, first_token_out_address, first_pool_address, @@ -38,7 +44,10 @@ def test_swaps( third_token_in_address, third_token_out_address, third_pool_address, - ] = get_addresses(12) + fourth_token_in_address, + fourth_token_out_address, + first_converter_address, + ] = get_addresses(16) first_token_in_amount = 10 first_token_out_amount = 20 @@ -46,6 +55,8 @@ def test_swaps( second_token_out_amount = 40 third_token_in_amount = 50 third_token_out_amount = 60 + fourth_token_in_amount = 70 + fourth_token_out_amount = 80 traces = [ make_unknown_trace(block_number, first_transaction_hash, []), @@ -139,11 +150,41 @@ def test_swaps( recipient_address=bob_address, recipient_input_key="recipient", ), + make_transfer_trace( + block_number, + fourth_transaction_hash, + trace_address=[2], + from_address=danielle_address, + to_address=first_converter_address, + token_address=fourth_token_in_address, + amount=fourth_token_in_amount, + ), + make_transfer_trace( + block_number, + fourth_transaction_hash, + trace_address=[1, 2], + from_address=first_converter_address, + to_address=danielle_address, + token_address=fourth_token_out_address, + amount=fourth_token_out_amount, + ), + make_swap_trace( + block_number, + fourth_transaction_hash, + trace_address=[], + from_address=danielle_address, + contract_address=BANCOR_NETWORK_CONTRACT_ADDRESS, + abi_name=BANCOR_NETWORK_ABI_NAME, + protocol=Protocol.bancor, + function_signature="convertByPath(address[],uint256,uint256,address,address,uint256)", + recipient_address=danielle_address, + recipient_input_key="recipient", + ), ] swaps = get_swaps(traces) - assert len(swaps) == 3 + assert len(swaps) == 4 for swap in swaps: if swap.abi_name == UNISWAP_V2_PAIR_ABI_NAME: @@ -152,6 +193,8 @@ def test_swaps( uni_v3_swap = swap elif swap.abi_name == BALANCER_V1_POOL_ABI_NAME: bal_v1_swap = swap + elif swap.abi_name == BANCOR_NETWORK_ABI_NAME: + bancor_swap = swap else: assert False @@ -193,3 +236,16 @@ def test_swaps( assert bal_v1_swap.token_in_amount == third_token_in_amount assert bal_v1_swap.token_out_address == third_token_out_address assert bal_v1_swap.token_out_amount == third_token_out_amount + + assert bancor_swap.abi_name == BANCOR_NETWORK_ABI_NAME + assert bancor_swap.transaction_hash == fourth_transaction_hash + assert bancor_swap.block_number == block_number + assert bancor_swap.trace_address == [] + assert bancor_swap.protocol == Protocol.bancor + assert bancor_swap.contract_address == BANCOR_NETWORK_CONTRACT_ADDRESS + assert bancor_swap.from_address == danielle_address + assert bancor_swap.to_address == danielle_address + assert bancor_swap.token_in_address == fourth_token_in_address + assert bancor_swap.token_in_amount == fourth_token_in_amount + assert bancor_swap.token_out_address == fourth_token_out_address + assert bancor_swap.token_out_amount == fourth_token_out_amount From 7d7f78bfb12ef2183a66658aa577faf5b10f0b0d Mon Sep 17 00:00:00 2001 From: Shea Ketsdever Date: Sun, 28 Nov 2021 16:02:41 -0800 Subject: [PATCH 18/26] Fix int<>timestamp bug --- mev_inspect/crud/blocks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mev_inspect/crud/blocks.py b/mev_inspect/crud/blocks.py index 3639ba8..c634795 100644 --- a/mev_inspect/crud/blocks.py +++ b/mev_inspect/crud/blocks.py @@ -1,3 +1,4 @@ +import datetime from mev_inspect.schemas.blocks import Block @@ -20,7 +21,7 @@ def write_block( "INSERT INTO blocks (block_number, block_timestamp) VALUES (:block_number, :block_timestamp)", params={ "block_number": block.block_number, - "block_timestamp": block.block_timestamp, + "block_timestamp": datetime.datetime.fromtimestamp(block.block_timestamp), }, ) db_session.commit() From ed94e717153cbe5a735967b2bc4a48d01694f8c0 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Tue, 30 Nov 2021 12:54:07 -0500 Subject: [PATCH 19/26] Fix timestamp writing in blocks --- mev_inspect/crud/blocks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mev_inspect/crud/blocks.py b/mev_inspect/crud/blocks.py index 3639ba8..90ead13 100644 --- a/mev_inspect/crud/blocks.py +++ b/mev_inspect/crud/blocks.py @@ -1,3 +1,5 @@ +from datetime import datetime + from mev_inspect.schemas.blocks import Block @@ -20,7 +22,7 @@ def write_block( "INSERT INTO blocks (block_number, block_timestamp) VALUES (:block_number, :block_timestamp)", params={ "block_number": block.block_number, - "block_timestamp": block.block_timestamp, + "block_timestamp": datetime.fromtimestamp(block.block_timestamp), }, ) db_session.commit() From 1fb65bacc144eb31652bf1247481dcd665cd75f8 Mon Sep 17 00:00:00 2001 From: Taarush Vemulapalli Date: Thu, 2 Dec 2021 14:19:32 -0500 Subject: [PATCH 20/26] Compound backfilling/removed network calls (#125) * Removes `collateral_token_address` from both aave/comp for consistency --- .../b9fa1ecc9929_remove_liq_column.py | 27 +++++++++++ mev_inspect/aave_liquidations.py | 1 - mev_inspect/compound_liquidations.py | 48 +------------------ mev_inspect/liquidations.py | 4 +- mev_inspect/models/liquidations.py | 1 - mev_inspect/schemas/liquidations.py | 1 - tests/liquidation_test.py | 8 ---- tests/test_compound.py | 16 ++----- 8 files changed, 37 insertions(+), 69 deletions(-) create mode 100644 alembic/versions/b9fa1ecc9929_remove_liq_column.py diff --git a/alembic/versions/b9fa1ecc9929_remove_liq_column.py b/alembic/versions/b9fa1ecc9929_remove_liq_column.py new file mode 100644 index 0000000..7fb1a08 --- /dev/null +++ b/alembic/versions/b9fa1ecc9929_remove_liq_column.py @@ -0,0 +1,27 @@ +"""Remove collateral_token_address column + +Revision ID: b9fa1ecc9929 +Revises: 04b76ab1d2af +Create Date: 2021-12-01 23:32:40.574108 + +""" +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "b9fa1ecc9929" +down_revision = "04b76ab1d2af" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_column("liquidations", "collateral_token_address") + + +def downgrade(): + op.add_column( + "liquidations", + sa.Column("collateral_token_address", sa.String(256), nullable=False), + ) diff --git a/mev_inspect/aave_liquidations.py b/mev_inspect/aave_liquidations.py index 1162028..d90a355 100644 --- a/mev_inspect/aave_liquidations.py +++ b/mev_inspect/aave_liquidations.py @@ -66,7 +66,6 @@ def get_aave_liquidations( liquidations.append( Liquidation( liquidated_user=trace.inputs["_user"], - collateral_token_address=trace.inputs["_collateral"], debt_token_address=trace.inputs["_reserve"], liquidator_user=liquidator, debt_purchase_amount=trace.inputs["_purchaseAmount"], diff --git a/mev_inspect/compound_liquidations.py b/mev_inspect/compound_liquidations.py index daae6ac..bc13252 100644 --- a/mev_inspect/compound_liquidations.py +++ b/mev_inspect/compound_liquidations.py @@ -1,5 +1,4 @@ -from typing import Dict, List, Optional -from web3 import Web3 +from typing import List, Optional from mev_inspect.traces import get_child_traces from mev_inspect.schemas.traces import ( @@ -9,44 +8,15 @@ from mev_inspect.schemas.traces import ( ) from mev_inspect.schemas.liquidations import Liquidation -from mev_inspect.abi import get_raw_abi -from mev_inspect.transfers import ETH_TOKEN_ADDRESS V2_COMPTROLLER_ADDRESS = "0x3d9819210A31b4961b30EF54bE2aeD79B9c9Cd3B" V2_C_ETHER = "0x4Ddc2D193948926D02f9B1fE9e1daa0718270ED5" CREAM_COMPTROLLER_ADDRESS = "0x3d5BC3c8d13dcB8bF317092d84783c2697AE9258" CREAM_CR_ETHER = "0xD06527D5e56A3495252A528C4987003b712860eE" -# helper, only queried once in the beginning (inspect_block) -def fetch_all_underlying_markets(w3: Web3, protocol: Protocol) -> Dict[str, str]: - if protocol == Protocol.compound_v2: - c_ether = V2_C_ETHER - address = V2_COMPTROLLER_ADDRESS - elif protocol == Protocol.cream: - c_ether = CREAM_CR_ETHER - address = CREAM_COMPTROLLER_ADDRESS - else: - raise ValueError(f"No Comptroller found for {protocol}") - token_mapping = {} - comptroller_abi = get_raw_abi("Comptroller", Protocol.compound_v2) - comptroller_instance = w3.eth.contract(address=address, abi=comptroller_abi) - markets = comptroller_instance.functions.getAllMarkets().call() - token_abi = get_raw_abi("CToken", Protocol.compound_v2) - for token in markets: - # make an exception for cETH (as it has no .underlying()) - if token != c_ether: - token_instance = w3.eth.contract(address=token, abi=token_abi) - underlying_token = token_instance.functions.underlying().call() - token_mapping[ - token.lower() - ] = underlying_token.lower() # make k:v lowercase for consistancy - return token_mapping - def get_compound_liquidations( traces: List[ClassifiedTrace], - collateral_by_c_token_address: Dict[str, str], - collateral_by_cr_token_address: Dict[str, str], ) -> List[Liquidation]: """Inspect list of classified traces and identify liquidation""" @@ -67,23 +37,13 @@ def get_compound_liquidations( trace.transaction_hash, trace.trace_address, traces ) seize_trace = _get_seize_call(child_traces) - underlying_markets = {} - if trace.protocol == Protocol.compound_v2: - underlying_markets = collateral_by_c_token_address - elif trace.protocol == Protocol.cream: - underlying_markets = collateral_by_cr_token_address - if ( - seize_trace is not None - and seize_trace.inputs is not None - and len(underlying_markets) != 0 - ): + if seize_trace is not None and seize_trace.inputs is not None: c_token_collateral = trace.inputs["cTokenCollateral"] if trace.abi_name == "CEther": liquidations.append( Liquidation( liquidated_user=trace.inputs["borrower"], - collateral_token_address=ETH_TOKEN_ADDRESS, # WETH since all cEther liquidations provide Ether debt_token_address=c_token_collateral, liquidator_user=seize_trace.inputs["liquidator"], debt_purchase_amount=trace.value, @@ -97,13 +57,9 @@ def get_compound_liquidations( elif ( trace.abi_name == "CToken" ): # cToken liquidations where liquidator pays back via token transfer - c_token_address = trace.to_address liquidations.append( Liquidation( liquidated_user=trace.inputs["borrower"], - collateral_token_address=underlying_markets[ - c_token_address - ], debt_token_address=c_token_collateral, liquidator_user=seize_trace.inputs["liquidator"], debt_purchase_amount=trace.inputs["repayAmount"], diff --git a/mev_inspect/liquidations.py b/mev_inspect/liquidations.py index 287431d..37f282a 100644 --- a/mev_inspect/liquidations.py +++ b/mev_inspect/liquidations.py @@ -1,6 +1,7 @@ from typing import List from mev_inspect.aave_liquidations import get_aave_liquidations +from mev_inspect.compound_liquidations import get_compound_liquidations from mev_inspect.schemas.traces import ( ClassifiedTrace, Classification, @@ -20,4 +21,5 @@ def get_liquidations( classified_traces: List[ClassifiedTrace], ) -> List[Liquidation]: aave_liquidations = get_aave_liquidations(classified_traces) - return aave_liquidations + comp_liquidations = get_compound_liquidations(classified_traces) + return aave_liquidations + comp_liquidations diff --git a/mev_inspect/models/liquidations.py b/mev_inspect/models/liquidations.py index 7f50175..b1e8c8d 100644 --- a/mev_inspect/models/liquidations.py +++ b/mev_inspect/models/liquidations.py @@ -8,7 +8,6 @@ class LiquidationModel(Base): liquidated_user = Column(String, nullable=False) liquidator_user = Column(String, nullable=False) - collateral_token_address = Column(String, nullable=False) debt_token_address = Column(String, nullable=False) debt_purchase_amount = Column(Numeric, nullable=False) received_amount = Column(Numeric, nullable=False) diff --git a/mev_inspect/schemas/liquidations.py b/mev_inspect/schemas/liquidations.py index 96f16b7..970912f 100644 --- a/mev_inspect/schemas/liquidations.py +++ b/mev_inspect/schemas/liquidations.py @@ -6,7 +6,6 @@ from mev_inspect.schemas.traces import Protocol class Liquidation(BaseModel): liquidated_user: str liquidator_user: str - collateral_token_address: str debt_token_address: str debt_purchase_amount: int received_amount: int diff --git a/tests/liquidation_test.py b/tests/liquidation_test.py index 8600d20..78bd2fa 100644 --- a/tests/liquidation_test.py +++ b/tests/liquidation_test.py @@ -19,7 +19,6 @@ def test_single_weth_liquidation(): Liquidation( liquidated_user="0xd16404ca0a74a15e66d8ad7c925592fb02422ffe", liquidator_user="0x19256c009781bc2d1545db745af6dfd30c7e9cfa", - collateral_token_address="0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", debt_token_address="0xdac17f958d2ee523a2206206994597c13d831ec7", debt_purchase_amount=26503300291, received_amount=8182733924513576561, @@ -50,7 +49,6 @@ def test_single_liquidation(): Liquidation( liquidated_user="0x8d8d912fe4db5917da92d14fea05225b803c359c", liquidator_user="0xf2d9e54f0e317b8ac94825b2543908e7552fe9c7", - collateral_token_address="0x80fb784b7ed66730e8b1dbd9820afd29931aab03", debt_token_address="0xdac17f958d2ee523a2206206994597c13d831ec7", debt_purchase_amount=1069206535, received_amount=2657946947610159065393, @@ -81,7 +79,6 @@ def test_single_liquidation_with_atoken_payback(): Liquidation( liquidated_user="0x3d2b6eacd1bca51af57ed8b3ff9ef0bd8ee8c56d", liquidator_user="0x887668f2dc9612280243f2a6ef834cecf456654e", - collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca", debt_token_address="0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", debt_purchase_amount=767615458043667978, received_amount=113993647930952952550, @@ -111,7 +108,6 @@ def test_multiple_liquidations_in_block(): liquidation1 = Liquidation( liquidated_user="0x6c6541ae8a7c6a6f968124a5ff2feac8f0c7875b", liquidator_user="0x7185e240d8e9e2d692cbc68d30eecf965e9a7feb", - collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca", debt_token_address="0x4fabb145d64652a948d72533023f6e7a623c7c53", debt_purchase_amount=457700000000000000000, received_amount=10111753901939162887, @@ -125,7 +121,6 @@ def test_multiple_liquidations_in_block(): liquidation2 = Liquidation( liquidated_user="0x6c6541ae8a7c6a6f968124a5ff2feac8f0c7875b", liquidator_user="0x7185e240d8e9e2d692cbc68d30eecf965e9a7feb", - collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca", debt_token_address="0x0000000000085d4780b73119b644ae5ecd22b376", debt_purchase_amount=497030000000000000000, received_amount=21996356316098208090, @@ -139,7 +134,6 @@ def test_multiple_liquidations_in_block(): liquidation3 = Liquidation( liquidated_user="0xda874f844389df33c0fad140df4970fe1b366726", liquidator_user="0x7185e240d8e9e2d692cbc68d30eecf965e9a7feb", - collateral_token_address="0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2", debt_token_address="0x57ab1ec28d129707052df4df418d58a2d46d5f51", debt_purchase_amount=447810000000000000000, received_amount=121531358145247546, @@ -169,7 +163,6 @@ def test_liquidations_with_eth_transfer(): liquidation1 = Liquidation( liquidated_user="0xad346c7762f74c78da86d2941c6eb546e316fbd0", liquidator_user="0x27239549dd40e1d60f5b80b0c4196923745b1fd2", - collateral_token_address=ETH_TOKEN_ADDRESS, debt_token_address="0x514910771af9ca656af840dff83e8264ecf986ca", debt_purchase_amount=1809152000000000000, received_amount=15636807387264000, @@ -183,7 +176,6 @@ def test_liquidations_with_eth_transfer(): liquidation2 = Liquidation( liquidated_user="0xad346c7762f74c78da86d2941c6eb546e316fbd0", liquidator_user="0x27239549dd40e1d60f5b80b0c4196923745b1fd2", - collateral_token_address=ETH_TOKEN_ADDRESS, debt_token_address="0x514910771af9ca656af840dff83e8264ecf986ca", debt_purchase_amount=1809152000000000000, received_amount=8995273139160873, diff --git a/tests/test_compound.py b/tests/test_compound.py index 6d0012e..1073b0e 100644 --- a/tests/test_compound.py +++ b/tests/test_compound.py @@ -2,7 +2,6 @@ from mev_inspect.compound_liquidations import get_compound_liquidations from mev_inspect.schemas.liquidations import Liquidation from mev_inspect.schemas.traces import Protocol from mev_inspect.classifiers.trace import TraceClassifier -from mev_inspect.transfers import ETH_TOKEN_ADDRESS from tests.utils import load_test_block, load_comp_markets, load_cream_markets comp_markets = load_comp_markets() @@ -19,7 +18,6 @@ def test_c_ether_liquidations(): Liquidation( liquidated_user="0xb5535a3681cf8d5431b8acfd779e2f79677ecce9", liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef", - collateral_token_address=ETH_TOKEN_ADDRESS, debt_token_address="0x39aa39c021dfbae8fac545936693ac917d5e7563", debt_purchase_amount=268066492249420078, received_amount=4747650169097, @@ -32,7 +30,7 @@ def test_c_ether_liquidations(): block = load_test_block(block_number) trace_classifier = TraceClassifier() classified_traces = trace_classifier.classify(block.traces) - result = get_compound_liquidations(classified_traces, comp_markets, cream_markets) + result = get_compound_liquidations(classified_traces) assert result == liquidations block_number = 13207907 @@ -44,7 +42,6 @@ def test_c_ether_liquidations(): Liquidation( liquidated_user="0x45df6f00166c3fb77dc16b9e47ff57bc6694e898", liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef", - collateral_token_address=ETH_TOKEN_ADDRESS, debt_token_address="0x35a18000230da775cac24873d00ff85bccded550", debt_purchase_amount=414547860568297082, received_amount=321973320649, @@ -58,7 +55,7 @@ def test_c_ether_liquidations(): block = load_test_block(block_number) trace_classifier = TraceClassifier() classified_traces = trace_classifier.classify(block.traces) - result = get_compound_liquidations(classified_traces, comp_markets, cream_markets) + result = get_compound_liquidations(classified_traces) assert result == liquidations block_number = 13298725 @@ -70,7 +67,6 @@ def test_c_ether_liquidations(): Liquidation( liquidated_user="0xacbcf5d2970eef25f02a27e9d9cd31027b058b9b", liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef", - collateral_token_address=ETH_TOKEN_ADDRESS, debt_token_address="0x35a18000230da775cac24873d00ff85bccded550", debt_purchase_amount=1106497772527562662, received_amount=910895850496, @@ -83,7 +79,7 @@ def test_c_ether_liquidations(): block = load_test_block(block_number) trace_classifier = TraceClassifier() classified_traces = trace_classifier.classify(block.traces) - result = get_compound_liquidations(classified_traces, comp_markets, cream_markets) + result = get_compound_liquidations(classified_traces) assert result == liquidations @@ -97,7 +93,6 @@ def test_c_token_liquidation(): Liquidation( liquidated_user="0xacdd5528c1c92b57045041b5278efa06cdade4d8", liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef", - collateral_token_address="0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", debt_token_address="0x70e36f6bf80a52b3b46b3af8e106cc0ed743e8e4", debt_purchase_amount=1207055531, received_amount=21459623305, @@ -110,7 +105,7 @@ def test_c_token_liquidation(): block = load_test_block(block_number) trace_classifier = TraceClassifier() classified_traces = trace_classifier.classify(block.traces) - result = get_compound_liquidations(classified_traces, comp_markets, cream_markets) + result = get_compound_liquidations(classified_traces) assert result == liquidations @@ -124,7 +119,6 @@ def test_cream_token_liquidation(): Liquidation( liquidated_user="0x46bf9479dc569bc796b7050344845f6564d45fba", liquidator_user="0xa2863cad9c318669660eb4eca8b3154b90fb4357", - collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca", debt_token_address="0x44fbebd2f576670a6c33f6fc0b00aa8c5753b322", debt_purchase_amount=14857434973806369550, received_amount=1547215810826, @@ -137,5 +131,5 @@ def test_cream_token_liquidation(): block = load_test_block(block_number) trace_classifier = TraceClassifier() classified_traces = trace_classifier.classify(block.traces) - result = get_compound_liquidations(classified_traces, comp_markets, cream_markets) + result = get_compound_liquidations(classified_traces) assert result == liquidations From 37e6900f46072dfcb7f8eb0560d50d923e003757 Mon Sep 17 00:00:00 2001 From: Shea Ketsdever Date: Thu, 2 Dec 2021 21:08:45 -0500 Subject: [PATCH 21/26] Rename create_swap functions --- mev_inspect/classifiers/helpers.py | 4 ++-- mev_inspect/classifiers/specs/balancer.py | 4 ++-- mev_inspect/classifiers/specs/bancor.py | 4 ++-- mev_inspect/classifiers/specs/curve.py | 4 ++-- mev_inspect/classifiers/specs/uniswap.py | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mev_inspect/classifiers/helpers.py b/mev_inspect/classifiers/helpers.py index f120005..064267c 100644 --- a/mev_inspect/classifiers/helpers.py +++ b/mev_inspect/classifiers/helpers.py @@ -6,7 +6,7 @@ from mev_inspect.schemas.transfers import Transfer, ETH_TOKEN_ADDRESS from mev_inspect.schemas.traces import DecodedCallTrace, ClassifiedTrace -def create_swap_from_transfers( +def create_swap_from_pool_transfers( trace: DecodedCallTrace, recipient_address: str, prior_transfers: List[Transfer], @@ -55,7 +55,7 @@ def create_swap_from_transfers( ) -def create_swap_from_transfers_not_including_pool( +def create_swap_from_recipient_transfers( trace: DecodedCallTrace, pool_address: str, recipient_address: str, diff --git a/mev_inspect/classifiers/specs/balancer.py b/mev_inspect/classifiers/specs/balancer.py index 89ae1f1..c3ad21a 100644 --- a/mev_inspect/classifiers/specs/balancer.py +++ b/mev_inspect/classifiers/specs/balancer.py @@ -9,7 +9,7 @@ from mev_inspect.schemas.classifiers import ( ClassifierSpec, SwapClassifier, ) -from mev_inspect.classifiers.helpers import create_swap_from_transfers +from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers BALANCER_V1_POOL_ABI_NAME = "BPool" @@ -24,7 +24,7 @@ class BalancerSwapClassifier(SwapClassifier): recipient_address = trace.from_address - swap = create_swap_from_transfers( + swap = create_swap_from_pool_transfers( trace, recipient_address, prior_transfers, child_transfers ) return swap diff --git a/mev_inspect/classifiers/specs/bancor.py b/mev_inspect/classifiers/specs/bancor.py index e8b15f6..f4a8663 100644 --- a/mev_inspect/classifiers/specs/bancor.py +++ b/mev_inspect/classifiers/specs/bancor.py @@ -10,7 +10,7 @@ from mev_inspect.schemas.classifiers import ( SwapClassifier, ) from mev_inspect.classifiers.helpers import ( - create_swap_from_transfers_not_including_pool, + create_swap_from_recipient_transfers, ) BANCOR_NETWORK_ABI_NAME = "BancorNetwork" @@ -26,7 +26,7 @@ class BancorSwapClassifier(SwapClassifier): ) -> Optional[Swap]: recipient_address = trace.from_address - swap = create_swap_from_transfers_not_including_pool( + swap = create_swap_from_recipient_transfers( trace, BANCOR_NETWORK_CONTRACT_ADDRESS, recipient_address, diff --git a/mev_inspect/classifiers/specs/curve.py b/mev_inspect/classifiers/specs/curve.py index 2762725..a395757 100644 --- a/mev_inspect/classifiers/specs/curve.py +++ b/mev_inspect/classifiers/specs/curve.py @@ -10,7 +10,7 @@ from mev_inspect.schemas.classifiers import ( ClassifierSpec, SwapClassifier, ) -from mev_inspect.classifiers.helpers import create_swap_from_transfers +from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers class CurveSwapClassifier(SwapClassifier): @@ -23,7 +23,7 @@ class CurveSwapClassifier(SwapClassifier): recipient_address = trace.from_address - swap = create_swap_from_transfers( + swap = create_swap_from_pool_transfers( trace, recipient_address, prior_transfers, child_transfers ) return swap diff --git a/mev_inspect/classifiers/specs/uniswap.py b/mev_inspect/classifiers/specs/uniswap.py index ce8b634..489078c 100644 --- a/mev_inspect/classifiers/specs/uniswap.py +++ b/mev_inspect/classifiers/specs/uniswap.py @@ -9,7 +9,7 @@ from mev_inspect.schemas.classifiers import ( ClassifierSpec, SwapClassifier, ) -from mev_inspect.classifiers.helpers import create_swap_from_transfers +from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers UNISWAP_V2_PAIR_ABI_NAME = "UniswapV2Pair" @@ -26,7 +26,7 @@ class UniswapV3SwapClassifier(SwapClassifier): recipient_address = trace.inputs.get("recipient", trace.from_address) - swap = create_swap_from_transfers( + swap = create_swap_from_pool_transfers( trace, recipient_address, prior_transfers, child_transfers ) return swap @@ -42,7 +42,7 @@ class UniswapV2SwapClassifier(SwapClassifier): recipient_address = trace.inputs.get("to", trace.from_address) - swap = create_swap_from_transfers( + swap = create_swap_from_pool_transfers( trace, recipient_address, prior_transfers, child_transfers ) return swap From 622cf9319ed6b7105c233c4a409a615a24131fad Mon Sep 17 00:00:00 2001 From: Robert Miller Date: Sat, 4 Dec 2021 20:31:46 -0500 Subject: [PATCH 22/26] style: formatting --- alembic/versions/7cf0eeb41da0_add_punk_bids.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alembic/versions/7cf0eeb41da0_add_punk_bids.py b/alembic/versions/7cf0eeb41da0_add_punk_bids.py index fabf2f6..0e93350 100644 --- a/alembic/versions/7cf0eeb41da0_add_punk_bids.py +++ b/alembic/versions/7cf0eeb41da0_add_punk_bids.py @@ -26,7 +26,7 @@ def upgrade(): sa.Column("from_address", sa.String(256), nullable=False), sa.Column("punk_index", sa.Numeric, nullable=False), sa.Column("price", sa.Numeric, nullable=False), - sa.PrimaryKeyConstraint("transaction_hash", "trace_address"), + sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"), ) From 478f9bafa504dedc406aa11f546474c90d5f485b Mon Sep 17 00:00:00 2001 From: Robert Miller Date: Sat, 4 Dec 2021 20:32:54 -0500 Subject: [PATCH 23/26] style: formatting --- alembic/versions/070819d86587_create_punk_snipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alembic/versions/070819d86587_create_punk_snipe.py b/alembic/versions/070819d86587_create_punk_snipe.py index fb50166..f6b5fcb 100644 --- a/alembic/versions/070819d86587_create_punk_snipe.py +++ b/alembic/versions/070819d86587_create_punk_snipe.py @@ -27,7 +27,7 @@ def upgrade(): sa.Column("punk_index", sa.Numeric, nullable=False), sa.Column("min_acceptance_price", sa.Numeric, nullable=False), sa.Column("acceptance_price", sa.Numeric, nullable=False), - sa.PrimaryKeyConstraint("transaction_hash", "trace_address"), + sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"), ) From be19c4227587908b541794e8985fa5208fb25bd4 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Mon, 6 Dec 2021 11:52:41 -0500 Subject: [PATCH 24/26] add start and end route check --- mev_inspect/arbitrages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mev_inspect/arbitrages.py b/mev_inspect/arbitrages.py index c13c8fe..5819dde 100644 --- a/mev_inspect/arbitrages.py +++ b/mev_inspect/arbitrages.py @@ -95,6 +95,7 @@ def _get_all_start_end_swaps(swaps: List[Swap]) -> List[Tuple[Swap, Swap]]: == potential_end_swap.token_out_address and potential_start_swap.from_address == potential_end_swap.to_address and not potential_start_swap.from_address in pool_addrs + and potential_start_swap != potential_end_swap ): valid_start_ends.append((potential_start_swap, potential_end_swap)) return valid_start_ends From d2a181477466a981eaac4e0db0b407234f1d6955 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Mon, 6 Dec 2021 15:18:32 -0500 Subject: [PATCH 25/26] skip start swap --- mev_inspect/arbitrages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mev_inspect/arbitrages.py b/mev_inspect/arbitrages.py index 5819dde..90a46d0 100644 --- a/mev_inspect/arbitrages.py +++ b/mev_inspect/arbitrages.py @@ -88,14 +88,14 @@ def _get_all_start_end_swaps(swaps: List[Swap]) -> List[Tuple[Swap, Swap]]: """ pool_addrs = [swap.contract_address for swap in swaps] valid_start_ends: List[Tuple[Swap, Swap]] = [] - for potential_start_swap in swaps: - for potential_end_swap in swaps: + for index, potential_start_swap in enumerate(swaps): + remaining_swaps = swaps[:index] + swaps[index + 1 :] + for potential_end_swap in remaining_swaps: if ( potential_start_swap.token_in_address == potential_end_swap.token_out_address and potential_start_swap.from_address == potential_end_swap.to_address and not potential_start_swap.from_address in pool_addrs - and potential_start_swap != potential_end_swap ): valid_start_ends.append((potential_start_swap, potential_end_swap)) return valid_start_ends From 01c4024017c572b56fc83a099bf5d284f99cdb96 Mon Sep 17 00:00:00 2001 From: Robert Miller Date: Mon, 6 Dec 2021 16:13:47 -0500 Subject: [PATCH 26/26] style: formatting --- mev_inspect/crud/punks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mev_inspect/crud/punks.py b/mev_inspect/crud/punks.py index ccbc652..7126287 100644 --- a/mev_inspect/crud/punks.py +++ b/mev_inspect/crud/punks.py @@ -32,6 +32,7 @@ def write_punk_bids( db_session.bulk_save_objects(models) db_session.commit() + def delete_punk_snipes_for_block( db_session, block_number: int,