Merge branch 'main' into punk-database-work

Dockerfile

@@ -18,4 +18,5 @@ COPY . /app
 # easter eggs 😝
 RUN echo "PS1='🕵️:\[\033[1;36m\]\h \[\033[1;34m\]\W\[\033[0;35m\]\[\033[1;36m\]$ \[\033[0m\]'" >> ~/.bashrc
 
-ENTRYPOINT [ "/app/entrypoint.sh"]
+ENTRYPOINT [ "poetry" ]
+CMD [ "run", "python", "loop.py" ]

Tiltfile

@@ -1,5 +1,4 @@
 load("ext://helm_remote", "helm_remote")
-load("ext://restart_process", "docker_build_with_restart")
 load("ext://secret", "secret_from_dict")
 load("ext://configmap", "configmap_from_dict")
 
@@ -30,8 +29,7 @@ k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = {
 #     "host": "trace-db-postgresql",
 # }))
 
-docker_build_with_restart("mev-inspect-py", ".",
+docker_build("mev-inspect-py", ".",
-    entrypoint="/app/entrypoint.sh",
     live_update=[
         sync(".", "/app"),
         run("cd /app && poetry install",
@@ -41,6 +39,10 @@ docker_build_with_restart("mev-inspect-py", ".",
 k8s_yaml(helm('./k8s/mev-inspect', name='mev-inspect'))
 k8s_resource(workload="mev-inspect", resource_deps=["postgresql-postgresql"])
 
+# uncomment to enable price monitor
+# k8s_yaml(helm('./k8s/mev-inspect-prices', name='mev-inspect-prices'))
+# k8s_resource(workload="mev-inspect-prices", resource_deps=["postgresql-postgresql"])
+
 local_resource(
     'pg-port-forward',
     serve_cmd='kubectl port-forward --namespace default svc/postgresql 5432:5432',

alembic/versions/7cf0eeb41da0_add_punk_bids.py

@@ -0,0 +1,34 @@
+"""empty message
+
+Revision ID: 7cf0eeb41da0
+Revises: d498bdb0a641
+Create Date: 2021-11-26 20:27:28.936516
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "7cf0eeb41da0"
+down_revision = "d498bdb0a641"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "punk_bids",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("trace_address", sa.String(256), nullable=False),
+        sa.Column("from_address", sa.String(256), nullable=False),
+        sa.Column("punk_index", sa.Numeric, nullable=False),
+        sa.Column("price", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
+    )
+
+
+def downgrade():
+    op.drop_table("punk_bids")

alembic/versions/b9fa1ecc9929_remove_liq_column.py

@@ -0,0 +1,27 @@
+"""Remove collateral_token_address column
+
+Revision ID: b9fa1ecc9929
+Revises: 04b76ab1d2af
+Create Date: 2021-12-01 23:32:40.574108
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "b9fa1ecc9929"
+down_revision = "04b76ab1d2af"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_column("liquidations", "collateral_token_address")
+
+
+def downgrade():
+    op.add_column(
+        "liquidations",
+        sa.Column("collateral_token_address", sa.String(256), nullable=False),
+    )

cli.py

@@ -5,12 +5,15 @@ import sys
 import click
 
 from mev_inspect.concurrency import coro
+from mev_inspect.crud.prices import write_prices
 from mev_inspect.db import get_inspect_session, get_trace_session
 from mev_inspect.inspector import MEVInspector
+from mev_inspect.prices import fetch_all_supported_prices
 
 RPC_URL_ENV = "RPC_URL"
 
 logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logger = logging.getLogger(__name__)
 
 
 @click.group()
@@ -79,6 +82,18 @@ async def inspect_many_blocks_command(
     )
 
 
+@cli.command()
+@coro
+async def fetch_all_prices():
+    inspect_db_session = get_inspect_session()
+
+    logger.info("Fetching prices")
+    prices = await fetch_all_supported_prices()
+
+    logger.info("Writing prices")
+    write_prices(inspect_db_session, prices)
+
+
 def get_rpc_url() -> str:
     return os.environ["RPC_URL"]
 

entrypoint.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-python loop.py

k8s/mev-inspect-backfill/templates/job.yaml

@@ -21,8 +21,7 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
-          command:
+          args:
-            - poetry
             - run
             - inspect-many-blocks
             - {{ .Values.command.startBlockNumber | quote }}

k8s/mev-inspect-prices/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

k8s/mev-inspect-prices/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: mev-inspect-prices
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

k8s/mev-inspect-prices/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "mev-inspect-prices.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mev-inspect-prices.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "mev-inspect-prices.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "mev-inspect-prices.labels" -}}
+helm.sh/chart: {{ include "mev-inspect-prices.chart" . }}
+{{ include "mev-inspect-prices.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "mev-inspect-prices.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "mev-inspect-prices.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "mev-inspect-prices.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "mev-inspect-prices.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

k8s/mev-inspect-prices/templates/cronjob.yaml

@@ -0,0 +1,35 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "mev-inspect-prices.fullname" . }}
+spec:
+  schedule: "0 */1 * * *"
+  successfulJobsHistoryLimit: 0
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - name: {{ .Chart.Name }}
+              image: "{{ .Values.image.repository }}"
+              imagePullPolicy: {{ .Values.image.pullPolicy }}
+              args:
+              - run
+              - fetch-all-prices
+              env:
+              - name: POSTGRES_HOST
+                valueFrom:
+                  secretKeyRef:
+                    name: mev-inspect-db-credentials
+                    key: host
+              - name: POSTGRES_USER
+                valueFrom:
+                  secretKeyRef:
+                    name: mev-inspect-db-credentials
+                    key: username
+              - name: POSTGRES_PASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: mev-inspect-db-credentials
+                    key: password
+          restartPolicy: Never

k8s/mev-inspect-prices/values.yaml

@@ -0,0 +1,7 @@
+image:
+  repository: mev-inspect-py
+  pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""

k8s/mev-inspect/templates/deployment.yaml

@@ -30,6 +30,7 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args: ["run", "python", "loop.py"]
           livenessProbe:
             exec:
               command:

mev

@@ -56,6 +56,19 @@ case "$1" in
       echo "Fetching block $block_number"
       kubectl exec -ti deploy/mev-inspect -- poetry run fetch-block $block_number
   ;;
+  prices)
+        shift
+        case "$1" in
+          fetch-all)
+                echo "Running price fetch-all"
+                kubectl exec -ti deploy/mev-inspect -- \
+                    poetry run fetch-all-prices
+            ;;
+          *)
+            echo "prices usage: "$1" {fetch-all}"
+            exit 1
+        esac
+  ;;
   exec)
         shift
         kubectl exec -ti deploy/mev-inspect -- $@

mev_inspect/aave_liquidations.py

@@ -66,7 +66,6 @@ def get_aave_liquidations(
             liquidations.append(
                 Liquidation(
                     liquidated_user=trace.inputs["_user"],
-                    collateral_token_address=trace.inputs["_collateral"],
                     debt_token_address=trace.inputs["_reserve"],
                     liquidator_user=liquidator,
                     debt_purchase_amount=trace.inputs["_purchaseAmount"],

mev_inspect/arbitrages.py

@@ -88,8 +88,9 @@ def _get_all_start_end_swaps(swaps: List[Swap]) -> List[Tuple[Swap, Swap]]:
     """
     pool_addrs = [swap.contract_address for swap in swaps]
     valid_start_ends: List[Tuple[Swap, Swap]] = []
-    for potential_start_swap in swaps:
+    for index, potential_start_swap in enumerate(swaps):
-        for potential_end_swap in swaps:
+        remaining_swaps = swaps[:index] + swaps[index + 1 :]
+        for potential_end_swap in remaining_swaps:
             if (
                 potential_start_swap.token_in_address
                 == potential_end_swap.token_out_address

mev_inspect/classifiers/helpers.py

@@ -6,7 +6,7 @@ from mev_inspect.schemas.transfers import Transfer, ETH_TOKEN_ADDRESS
 from mev_inspect.schemas.traces import DecodedCallTrace, ClassifiedTrace
 
 
-def create_swap_from_transfers(
+def create_swap_from_pool_transfers(
     trace: DecodedCallTrace,
     recipient_address: str,
     prior_transfers: List[Transfer],
@@ -55,6 +55,43 @@ def create_swap_from_transfers(
     )
 
 
+def create_swap_from_recipient_transfers(
+    trace: DecodedCallTrace,
+    pool_address: str,
+    recipient_address: str,
+    prior_transfers: List[Transfer],
+    child_transfers: List[Transfer],
+) -> Optional[Swap]:
+    transfers_from_recipient = _filter_transfers(
+        [*prior_transfers, *child_transfers], from_address=recipient_address
+    )
+    transfers_to_recipient = _filter_transfers(
+        child_transfers, to_address=recipient_address
+    )
+
+    if len(transfers_from_recipient) != 1 or len(transfers_to_recipient) != 1:
+        return None
+
+    transfer_in = transfers_from_recipient[0]
+    transfer_out = transfers_to_recipient[0]
+
+    return Swap(
+        abi_name=trace.abi_name,
+        transaction_hash=trace.transaction_hash,
+        block_number=trace.block_number,
+        trace_address=trace.trace_address,
+        contract_address=pool_address,
+        protocol=trace.protocol,
+        from_address=transfer_in.from_address,
+        to_address=transfer_out.to_address,
+        token_in_address=transfer_in.token_address,
+        token_in_amount=transfer_in.amount,
+        token_out_address=transfer_out.token_address,
+        token_out_amount=transfer_out.amount,
+        error=trace.error,
+    )
+
+
 def _build_eth_transfer(trace: ClassifiedTrace) -> Transfer:
     return Transfer(
         block_number=trace.block_number,

mev_inspect/classifiers/specs/__init__.py

@@ -12,6 +12,7 @@ from .zero_ex import ZEROX_CLASSIFIER_SPECS
 from .balancer import BALANCER_CLASSIFIER_SPECS
 from .compound import COMPOUND_CLASSIFIER_SPECS
 from .cryptopunks import CRYPTOPUNKS_CLASSIFIER_SPECS
+from .bancor import BANCOR_CLASSIFIER_SPECS
 
 ALL_CLASSIFIER_SPECS = (
     ERC20_CLASSIFIER_SPECS
@@ -23,6 +24,7 @@ ALL_CLASSIFIER_SPECS = (
     + BALANCER_CLASSIFIER_SPECS
     + COMPOUND_CLASSIFIER_SPECS
     + CRYPTOPUNKS_CLASSIFIER_SPECS
+    + BANCOR_CLASSIFIER_SPECS
 )
 
 _SPECS_BY_ABI_NAME_AND_PROTOCOL: Dict[

mev_inspect/classifiers/specs/balancer.py

@@ -9,7 +9,7 @@ from mev_inspect.schemas.classifiers import (
     ClassifierSpec,
     SwapClassifier,
 )
-from mev_inspect.classifiers.helpers import create_swap_from_transfers
+from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
 
 BALANCER_V1_POOL_ABI_NAME = "BPool"
 
@@ -24,7 +24,7 @@ class BalancerSwapClassifier(SwapClassifier):
 
         recipient_address = trace.from_address
 
-        swap = create_swap_from_transfers(
+        swap = create_swap_from_pool_transfers(
             trace, recipient_address, prior_transfers, child_transfers
         )
         return swap

mev_inspect/classifiers/specs/bancor.py

@@ -0,0 +1,48 @@
+from typing import Optional, List
+from mev_inspect.schemas.transfers import Transfer
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import (
+    DecodedCallTrace,
+    Protocol,
+)
+from mev_inspect.schemas.classifiers import (
+    ClassifierSpec,
+    SwapClassifier,
+)
+from mev_inspect.classifiers.helpers import (
+    create_swap_from_recipient_transfers,
+)
+
+BANCOR_NETWORK_ABI_NAME = "BancorNetwork"
+BANCOR_NETWORK_CONTRACT_ADDRESS = "0x2F9EC37d6CcFFf1caB21733BdaDEdE11c823cCB0"
+
+
+class BancorSwapClassifier(SwapClassifier):
+    @staticmethod
+    def parse_swap(
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
+        recipient_address = trace.from_address
+
+        swap = create_swap_from_recipient_transfers(
+            trace,
+            BANCOR_NETWORK_CONTRACT_ADDRESS,
+            recipient_address,
+            prior_transfers,
+            child_transfers,
+        )
+        return swap
+
+
+BANCOR_NETWORK_SPEC = ClassifierSpec(
+    abi_name=BANCOR_NETWORK_ABI_NAME,
+    protocol=Protocol.bancor,
+    classifiers={
+        "convertByPath(address[],uint256,uint256,address,address,uint256)": BancorSwapClassifier,
+    },
+    valid_contract_addresses=[BANCOR_NETWORK_CONTRACT_ADDRESS],
+)
+
+BANCOR_CLASSIFIER_SPECS = [BANCOR_NETWORK_SPEC]

mev_inspect/classifiers/specs/curve.py

@@ -10,7 +10,7 @@ from mev_inspect.schemas.classifiers import (
     ClassifierSpec,
     SwapClassifier,
 )
-from mev_inspect.classifiers.helpers import create_swap_from_transfers
+from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
 
 
 class CurveSwapClassifier(SwapClassifier):
@@ -23,7 +23,7 @@ class CurveSwapClassifier(SwapClassifier):
 
         recipient_address = trace.from_address
 
-        swap = create_swap_from_transfers(
+        swap = create_swap_from_pool_transfers(
             trace, recipient_address, prior_transfers, child_transfers
         )
         return swap

mev_inspect/classifiers/specs/uniswap.py

@@ -9,7 +9,7 @@ from mev_inspect.schemas.classifiers import (
     ClassifierSpec,
     SwapClassifier,
 )
-from mev_inspect.classifiers.helpers import create_swap_from_transfers
+from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
 
 
 UNISWAP_V2_PAIR_ABI_NAME = "UniswapV2Pair"
@@ -26,7 +26,7 @@ class UniswapV3SwapClassifier(SwapClassifier):
 
         recipient_address = trace.inputs.get("recipient", trace.from_address)
 
-        swap = create_swap_from_transfers(
+        swap = create_swap_from_pool_transfers(
             trace, recipient_address, prior_transfers, child_transfers
         )
         return swap
@@ -42,7 +42,7 @@ class UniswapV2SwapClassifier(SwapClassifier):
 
         recipient_address = trace.inputs.get("to", trace.from_address)
 
-        swap = create_swap_from_transfers(
+        swap = create_swap_from_pool_transfers(
             trace, recipient_address, prior_transfers, child_transfers
         )
         return swap

mev_inspect/coinbase.py

@@ -0,0 +1,25 @@
+import aiohttp
+
+from mev_inspect.classifiers.specs.weth import WETH_ADDRESS
+from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS
+from mev_inspect.schemas.coinbase import CoinbasePrices, CoinbasePricesResponse
+
+
+COINBASE_API_BASE = "https://www.coinbase.com/api/v2"
+COINBASE_TOKEN_NAME_BY_ADDRESS = {
+    WETH_ADDRESS: "weth",
+    ETH_TOKEN_ADDRESS: "ethereum",
+}
+
+
+async def fetch_coinbase_prices(token_address: str) -> CoinbasePrices:
+    if token_address not in COINBASE_TOKEN_NAME_BY_ADDRESS:
+        raise ValueError(f"Unsupported token_address {token_address}")
+
+    coinbase_token_name = COINBASE_TOKEN_NAME_BY_ADDRESS[token_address]
+    url = f"{COINBASE_API_BASE}/assets/prices/{coinbase_token_name}"
+
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url, params={"base": "USD"}) as response:
+            json_data = await response.json()
+            return CoinbasePricesResponse(**json_data).data.prices

mev_inspect/compound_liquidations.py

@@ -1,5 +1,4 @@
-from typing import Dict, List, Optional
+from typing import List, Optional
-from web3 import Web3
 
 from mev_inspect.traces import get_child_traces
 from mev_inspect.schemas.traces import (
@@ -9,44 +8,15 @@ from mev_inspect.schemas.traces import (
 )
 
 from mev_inspect.schemas.liquidations import Liquidation
-from mev_inspect.abi import get_raw_abi
-from mev_inspect.transfers import ETH_TOKEN_ADDRESS
 
 V2_COMPTROLLER_ADDRESS = "0x3d9819210A31b4961b30EF54bE2aeD79B9c9Cd3B"
 V2_C_ETHER = "0x4Ddc2D193948926D02f9B1fE9e1daa0718270ED5"
 CREAM_COMPTROLLER_ADDRESS = "0x3d5BC3c8d13dcB8bF317092d84783c2697AE9258"
 CREAM_CR_ETHER = "0xD06527D5e56A3495252A528C4987003b712860eE"
 
-# helper, only queried once in the beginning (inspect_block)
-def fetch_all_underlying_markets(w3: Web3, protocol: Protocol) -> Dict[str, str]:
-    if protocol == Protocol.compound_v2:
-        c_ether = V2_C_ETHER
-        address = V2_COMPTROLLER_ADDRESS
-    elif protocol == Protocol.cream:
-        c_ether = CREAM_CR_ETHER
-        address = CREAM_COMPTROLLER_ADDRESS
-    else:
-        raise ValueError(f"No Comptroller found for {protocol}")
-    token_mapping = {}
-    comptroller_abi = get_raw_abi("Comptroller", Protocol.compound_v2)
-    comptroller_instance = w3.eth.contract(address=address, abi=comptroller_abi)
-    markets = comptroller_instance.functions.getAllMarkets().call()
-    token_abi = get_raw_abi("CToken", Protocol.compound_v2)
-    for token in markets:
-        # make an exception for cETH (as it has no .underlying())
-        if token != c_ether:
-            token_instance = w3.eth.contract(address=token, abi=token_abi)
-            underlying_token = token_instance.functions.underlying().call()
-            token_mapping[
-                token.lower()
-            ] = underlying_token.lower()  # make k:v lowercase for consistancy
-    return token_mapping
-
 
 def get_compound_liquidations(
     traces: List[ClassifiedTrace],
-    collateral_by_c_token_address: Dict[str, str],
-    collateral_by_cr_token_address: Dict[str, str],
 ) -> List[Liquidation]:
 
     """Inspect list of classified traces and identify liquidation"""
@@ -67,23 +37,13 @@ def get_compound_liquidations(
                 trace.transaction_hash, trace.trace_address, traces
             )
             seize_trace = _get_seize_call(child_traces)
-            underlying_markets = {}
-            if trace.protocol == Protocol.compound_v2:
-                underlying_markets = collateral_by_c_token_address
-            elif trace.protocol == Protocol.cream:
-                underlying_markets = collateral_by_cr_token_address
 
-            if (
+            if seize_trace is not None and seize_trace.inputs is not None:
-                seize_trace is not None
-                and seize_trace.inputs is not None
-                and len(underlying_markets) != 0
-            ):
                 c_token_collateral = trace.inputs["cTokenCollateral"]
                 if trace.abi_name == "CEther":
                     liquidations.append(
                         Liquidation(
                             liquidated_user=trace.inputs["borrower"],
-                            collateral_token_address=ETH_TOKEN_ADDRESS,  # WETH since all cEther liquidations provide Ether
                             debt_token_address=c_token_collateral,
                             liquidator_user=seize_trace.inputs["liquidator"],
                             debt_purchase_amount=trace.value,
@@ -97,13 +57,9 @@ def get_compound_liquidations(
                 elif (
                     trace.abi_name == "CToken"
                 ):  # cToken liquidations where liquidator pays back via token transfer
-                    c_token_address = trace.to_address
                     liquidations.append(
                         Liquidation(
                             liquidated_user=trace.inputs["borrower"],
-                            collateral_token_address=underlying_markets[
-                                c_token_address
-                            ],
                             debt_token_address=c_token_collateral,
                             liquidator_user=seize_trace.inputs["liquidator"],
                             debt_purchase_amount=trace.inputs["repayAmount"],

mev_inspect/crud/blocks.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+
 from mev_inspect.schemas.blocks import Block
 
 
@@ -20,7 +22,7 @@ def write_block(
         "INSERT INTO blocks (block_number, block_timestamp) VALUES (:block_number, :block_timestamp)",
         params={
             "block_number": block.block_number,
-            "block_timestamp": block.block_timestamp,
+            "block_timestamp": datetime.fromtimestamp(block.block_timestamp),
         },
     )
     db_session.commit()

mev_inspect/crud/prices.py

@@ -0,0 +1,17 @@
+from typing import List
+
+from sqlalchemy.dialects.postgresql import insert
+
+from mev_inspect.models.prices import PriceModel
+from mev_inspect.schemas.prices import Price
+
+
+def write_prices(db_session, prices: List[Price]) -> None:
+    insert_statement = (
+        insert(PriceModel.__table__)
+        .values([price.dict() for price in prices])
+        .on_conflict_do_nothing()
+    )
+
+    db_session.execute(insert_statement)
+    db_session.commit()

mev_inspect/crud/punks.py

@@ -1,9 +1,36 @@
 import json
 from typing import List
 
-from mev_inspect.models.punk_snipes import PunkSnipeModel
+from mev_inspect.models.punks import (
+    PunkSnipeModel,
+    PunkBidModel,
+)
 from mev_inspect.schemas.punk_snipe import PunkSnipe
 
+from mev_inspect.schemas.punk_bid import PunkBid
+
+
+def delete_punk_bids_for_block(
+    db_session,
+    block_number: int,
+) -> None:
+    (
+        db_session.query(PunkBidModel)
+        .filter(PunkBidModel.block_number == block_number)
+        .delete()
+    )
+
+    db_session.commit()
+
+
+def write_punk_bids(
+    db_session,
+    punk_bids: List[PunkBid],
+) -> None:
+    models = [PunkBidModel(**json.loads(punk_bid.json())) for punk_bid in punk_bids]
+
+    db_session.bulk_save_objects(models)
+    db_session.commit()
 
 def delete_punk_snipes_for_block(
     db_session,

mev_inspect/inspect_block.py

@@ -12,8 +12,12 @@ from mev_inspect.crud.arbitrages import (
     write_arbitrages,
 )
 
-from mev_inspect.crud.punks import delete_punk_snipes_for_block, write_punk_snipes
+from mev_inspect.crud.punks import (
-
+    delete_punk_snipes_for_block,
+    write_punk_snipes,
+    delete_punk_bids_for_block,
+    write_punk_bids,
+)
 from mev_inspect.crud.blocks import (
     delete_block,
     write_block,
@@ -103,6 +107,9 @@ async def inspect_block(
     write_liquidations(inspect_db_session, liquidations)
 
     punk_bids = get_punk_bids(classified_traces)
+    delete_punk_bids_for_block(inspect_db_session, block_number)
+    write_punk_bids(inspect_db_session, punk_bids)
+
     punk_bid_acceptances = get_punk_bid_acceptances(classified_traces)
 
     punk_snipes = get_punk_snipes(punk_bids, punk_bid_acceptances)

mev_inspect/liquidations.py

@@ -1,6 +1,7 @@
 from typing import List
 
 from mev_inspect.aave_liquidations import get_aave_liquidations
+from mev_inspect.compound_liquidations import get_compound_liquidations
 from mev_inspect.schemas.traces import (
     ClassifiedTrace,
     Classification,
@@ -20,4 +21,5 @@ def get_liquidations(
     classified_traces: List[ClassifiedTrace],
 ) -> List[Liquidation]:
     aave_liquidations = get_aave_liquidations(classified_traces)
-    return aave_liquidations
+    comp_liquidations = get_compound_liquidations(classified_traces)
+    return aave_liquidations + comp_liquidations

mev_inspect/models/liquidations.py

@@ -8,7 +8,6 @@ class LiquidationModel(Base):
 
     liquidated_user = Column(String, nullable=False)
     liquidator_user = Column(String, nullable=False)
-    collateral_token_address = Column(String, nullable=False)
     debt_token_address = Column(String, nullable=False)
     debt_purchase_amount = Column(Numeric, nullable=False)
     received_amount = Column(Numeric, nullable=False)

mev_inspect/models/prices.py

@@ -0,0 +1,11 @@
+from sqlalchemy import Column, Numeric, String, TIMESTAMP
+
+from .base import Base
+
+
+class PriceModel(Base):
+    __tablename__ = "prices"
+
+    timestamp = Column(TIMESTAMP, nullable=False, primary_key=True)
+    usd_price = Column(Numeric, nullable=False)
+    token_address = Column(String, nullable=False, primary_key=True)

mev_inspect/models/punks.py

@@ -0,0 +1,26 @@
+from sqlalchemy import Column, Numeric, String, ARRAY, Integer
+
+from .base import Base
+
+
+class PunkSnipeModel(Base):
+    __tablename__ = "punk_snipes"
+
+    block_number = Column(Numeric, nullable=False)
+    transaction_hash = Column(String, primary_key=True)
+    trace_address = Column(ARRAY(Integer), primary_key=True)
+    from_address = Column(String, nullable=False)
+    punk_index = Column(Integer, nullable=False)
+    min_acceptance_price = Column(Numeric, nullable=False)
+    acceptance_price = Column(Numeric, nullable=False)
+
+
+class PunkBidModel(Base):
+    __tablename__ = "punk_bids"
+
+    block_number = Column(Numeric, nullable=False)
+    transaction_hash = Column(String, primary_key=True)
+    trace_address = Column(ARRAY(Integer), primary_key=True)
+    from_address = Column(String, nullable=False)
+    punk_index = Column(Integer, nullable=False)
+    price = Column(Numeric, nullable=False)

mev_inspect/prices.py

@@ -0,0 +1,29 @@
+from typing import List
+
+from mev_inspect.classifiers.specs.weth import WETH_ADDRESS
+from mev_inspect.coinbase import fetch_coinbase_prices
+from mev_inspect.schemas.prices import Price
+from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS
+
+
+SUPPORTED_TOKENS = [
+    WETH_ADDRESS,
+    ETH_TOKEN_ADDRESS,
+]
+
+
+async def fetch_all_supported_prices() -> List[Price]:
+    prices = []
+
+    for token_address in SUPPORTED_TOKENS:
+        coinbase_prices = await fetch_coinbase_prices(token_address)
+        for usd_price, timestamp_seconds in coinbase_prices.all.prices:
+            price = Price(
+                token_address=token_address,
+                usd_price=usd_price,
+                timestamp=timestamp_seconds,
+            )
+
+            prices.append(price)
+
+    return prices

mev_inspect/schemas/coinbase.py

@@ -0,0 +1,20 @@
+from typing import List, Tuple
+
+from pydantic import BaseModel
+
+
+class CoinbasePricesEntry(BaseModel):
+    # tuple of price and timestamp
+    prices: List[Tuple[float, int]]
+
+
+class CoinbasePrices(BaseModel):
+    all: CoinbasePricesEntry
+
+
+class CoinbasePricesDataResponse(BaseModel):
+    prices: CoinbasePrices
+
+
+class CoinbasePricesResponse(BaseModel):
+    data: CoinbasePricesDataResponse

mev_inspect/schemas/liquidations.py

@@ -6,7 +6,6 @@ from mev_inspect.schemas.traces import Protocol
 class Liquidation(BaseModel):
     liquidated_user: str
     liquidator_user: str
-    collateral_token_address: str
     debt_token_address: str
     debt_purchase_amount: int
     received_amount: int

mev_inspect/schemas/prices.py

@@ -0,0 +1,9 @@
+from datetime import datetime
+
+from pydantic import BaseModel
+
+
+class Price(BaseModel):
+    token_address: str
+    timestamp: datetime
+    usd_price: float

mev_inspect/schemas/traces.py

@@ -47,6 +47,7 @@ class Protocol(Enum):
     compound_v2 = "compound_v2"
     cream = "cream"
     cryptopunks = "cryptopunks"
+    bancor = "bancor"
 
 
 class ClassifiedTrace(Trace):

mev_inspect/schemas/utils.py

@@ -1,8 +1,8 @@
 import json
 
 from hexbytes import HexBytes
-from web3.datastructures import AttributeDict
 from pydantic import BaseModel
+from web3.datastructures import AttributeDict
 
 
 def to_camel(string: str) -> str:

pyproject.toml

@@ -34,6 +34,7 @@ build-backend = "poetry.core.masonry.api"
 inspect-block = 'cli:inspect_block_command'
 inspect-many-blocks = 'cli:inspect_many_blocks_command'
 fetch-block = 'cli:fetch_block_command'
+fetch-all-prices = 'cli:fetch_all_prices'
 
 [tool.black]
 exclude = '''

tests/liquidation_test.py

@@ -19,7 +19,6 @@ def test_single_weth_liquidation():
         Liquidation(
             liquidated_user="0xd16404ca0a74a15e66d8ad7c925592fb02422ffe",
             liquidator_user="0x19256c009781bc2d1545db745af6dfd30c7e9cfa",
-            collateral_token_address="0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2",
             debt_token_address="0xdac17f958d2ee523a2206206994597c13d831ec7",
             debt_purchase_amount=26503300291,
             received_amount=8182733924513576561,
@@ -50,7 +49,6 @@ def test_single_liquidation():
         Liquidation(
             liquidated_user="0x8d8d912fe4db5917da92d14fea05225b803c359c",
             liquidator_user="0xf2d9e54f0e317b8ac94825b2543908e7552fe9c7",
-            collateral_token_address="0x80fb784b7ed66730e8b1dbd9820afd29931aab03",
             debt_token_address="0xdac17f958d2ee523a2206206994597c13d831ec7",
             debt_purchase_amount=1069206535,
             received_amount=2657946947610159065393,
@@ -81,7 +79,6 @@ def test_single_liquidation_with_atoken_payback():
         Liquidation(
             liquidated_user="0x3d2b6eacd1bca51af57ed8b3ff9ef0bd8ee8c56d",
             liquidator_user="0x887668f2dc9612280243f2a6ef834cecf456654e",
-            collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
             debt_token_address="0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2",
             debt_purchase_amount=767615458043667978,
             received_amount=113993647930952952550,
@@ -111,7 +108,6 @@ def test_multiple_liquidations_in_block():
     liquidation1 = Liquidation(
         liquidated_user="0x6c6541ae8a7c6a6f968124a5ff2feac8f0c7875b",
         liquidator_user="0x7185e240d8e9e2d692cbc68d30eecf965e9a7feb",
-        collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
         debt_token_address="0x4fabb145d64652a948d72533023f6e7a623c7c53",
         debt_purchase_amount=457700000000000000000,
         received_amount=10111753901939162887,
@@ -125,7 +121,6 @@ def test_multiple_liquidations_in_block():
     liquidation2 = Liquidation(
         liquidated_user="0x6c6541ae8a7c6a6f968124a5ff2feac8f0c7875b",
         liquidator_user="0x7185e240d8e9e2d692cbc68d30eecf965e9a7feb",
-        collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
         debt_token_address="0x0000000000085d4780b73119b644ae5ecd22b376",
         debt_purchase_amount=497030000000000000000,
         received_amount=21996356316098208090,
@@ -139,7 +134,6 @@ def test_multiple_liquidations_in_block():
     liquidation3 = Liquidation(
         liquidated_user="0xda874f844389df33c0fad140df4970fe1b366726",
         liquidator_user="0x7185e240d8e9e2d692cbc68d30eecf965e9a7feb",
-        collateral_token_address="0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2",
         debt_token_address="0x57ab1ec28d129707052df4df418d58a2d46d5f51",
         debt_purchase_amount=447810000000000000000,
         received_amount=121531358145247546,
@@ -169,7 +163,6 @@ def test_liquidations_with_eth_transfer():
     liquidation1 = Liquidation(
         liquidated_user="0xad346c7762f74c78da86d2941c6eb546e316fbd0",
         liquidator_user="0x27239549dd40e1d60f5b80b0c4196923745b1fd2",
-        collateral_token_address=ETH_TOKEN_ADDRESS,
         debt_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
         debt_purchase_amount=1809152000000000000,
         received_amount=15636807387264000,
@@ -183,7 +176,6 @@ def test_liquidations_with_eth_transfer():
     liquidation2 = Liquidation(
         liquidated_user="0xad346c7762f74c78da86d2941c6eb546e316fbd0",
         liquidator_user="0x27239549dd40e1d60f5b80b0c4196923745b1fd2",
-        collateral_token_address=ETH_TOKEN_ADDRESS,
         debt_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
         debt_purchase_amount=1809152000000000000,
         received_amount=8995273139160873,

tests/test_compound.py

@@ -2,7 +2,6 @@ from mev_inspect.compound_liquidations import get_compound_liquidations
 from mev_inspect.schemas.liquidations import Liquidation
 from mev_inspect.schemas.traces import Protocol
 from mev_inspect.classifiers.trace import TraceClassifier
-from mev_inspect.transfers import ETH_TOKEN_ADDRESS
 from tests.utils import load_test_block, load_comp_markets, load_cream_markets
 
 comp_markets = load_comp_markets()
@@ -19,7 +18,6 @@ def test_c_ether_liquidations():
         Liquidation(
             liquidated_user="0xb5535a3681cf8d5431b8acfd779e2f79677ecce9",
             liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
-            collateral_token_address=ETH_TOKEN_ADDRESS,
             debt_token_address="0x39aa39c021dfbae8fac545936693ac917d5e7563",
             debt_purchase_amount=268066492249420078,
             received_amount=4747650169097,
@@ -32,7 +30,7 @@ def test_c_ether_liquidations():
     block = load_test_block(block_number)
     trace_classifier = TraceClassifier()
     classified_traces = trace_classifier.classify(block.traces)
-    result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
+    result = get_compound_liquidations(classified_traces)
     assert result == liquidations
 
     block_number = 13207907
@@ -44,7 +42,6 @@ def test_c_ether_liquidations():
         Liquidation(
             liquidated_user="0x45df6f00166c3fb77dc16b9e47ff57bc6694e898",
             liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
-            collateral_token_address=ETH_TOKEN_ADDRESS,
             debt_token_address="0x35a18000230da775cac24873d00ff85bccded550",
             debt_purchase_amount=414547860568297082,
             received_amount=321973320649,
@@ -58,7 +55,7 @@ def test_c_ether_liquidations():
     block = load_test_block(block_number)
     trace_classifier = TraceClassifier()
     classified_traces = trace_classifier.classify(block.traces)
-    result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
+    result = get_compound_liquidations(classified_traces)
     assert result == liquidations
 
     block_number = 13298725
@@ -70,7 +67,6 @@ def test_c_ether_liquidations():
         Liquidation(
             liquidated_user="0xacbcf5d2970eef25f02a27e9d9cd31027b058b9b",
             liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
-            collateral_token_address=ETH_TOKEN_ADDRESS,
             debt_token_address="0x35a18000230da775cac24873d00ff85bccded550",
             debt_purchase_amount=1106497772527562662,
             received_amount=910895850496,
@@ -83,7 +79,7 @@ def test_c_ether_liquidations():
     block = load_test_block(block_number)
     trace_classifier = TraceClassifier()
     classified_traces = trace_classifier.classify(block.traces)
-    result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
+    result = get_compound_liquidations(classified_traces)
     assert result == liquidations
 
 
@@ -97,7 +93,6 @@ def test_c_token_liquidation():
         Liquidation(
             liquidated_user="0xacdd5528c1c92b57045041b5278efa06cdade4d8",
             liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
-            collateral_token_address="0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48",
             debt_token_address="0x70e36f6bf80a52b3b46b3af8e106cc0ed743e8e4",
             debt_purchase_amount=1207055531,
             received_amount=21459623305,
@@ -110,7 +105,7 @@ def test_c_token_liquidation():
     block = load_test_block(block_number)
     trace_classifier = TraceClassifier()
     classified_traces = trace_classifier.classify(block.traces)
-    result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
+    result = get_compound_liquidations(classified_traces)
     assert result == liquidations
 
 
@@ -124,7 +119,6 @@ def test_cream_token_liquidation():
         Liquidation(
             liquidated_user="0x46bf9479dc569bc796b7050344845f6564d45fba",
             liquidator_user="0xa2863cad9c318669660eb4eca8b3154b90fb4357",
-            collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
             debt_token_address="0x44fbebd2f576670a6c33f6fc0b00aa8c5753b322",
             debt_purchase_amount=14857434973806369550,
             received_amount=1547215810826,
@@ -137,5 +131,5 @@ def test_cream_token_liquidation():
     block = load_test_block(block_number)
     trace_classifier = TraceClassifier()
     classified_traces = trace_classifier.classify(block.traces)
-    result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
+    result = get_compound_liquidations(classified_traces)
     assert result == liquidations

tests/test_swaps.py

@@ -4,6 +4,10 @@ from mev_inspect.classifiers.specs.uniswap import (
     UNISWAP_V2_PAIR_ABI_NAME,
     UNISWAP_V3_POOL_ABI_NAME,
 )
+from mev_inspect.classifiers.specs.bancor import (
+    BANCOR_NETWORK_ABI_NAME,
+    BANCOR_NETWORK_CONTRACT_ADDRESS,
+)
 from mev_inspect.schemas.traces import Protocol
 
 from .helpers import (
@@ -23,12 +27,14 @@ def test_swaps(
         first_transaction_hash,
         second_transaction_hash,
         third_transaction_hash,
-    ] = get_transaction_hashes(3)
+        fourth_transaction_hash,
+    ] = get_transaction_hashes(4)
 
     [
         alice_address,
         bob_address,
         carl_address,
+        danielle_address,
         first_token_in_address,
         first_token_out_address,
         first_pool_address,
@@ -38,7 +44,10 @@ def test_swaps(
         third_token_in_address,
         third_token_out_address,
         third_pool_address,
-    ] = get_addresses(12)
+        fourth_token_in_address,
+        fourth_token_out_address,
+        first_converter_address,
+    ] = get_addresses(16)
 
     first_token_in_amount = 10
     first_token_out_amount = 20
@@ -46,6 +55,8 @@ def test_swaps(
     second_token_out_amount = 40
     third_token_in_amount = 50
     third_token_out_amount = 60
+    fourth_token_in_amount = 70
+    fourth_token_out_amount = 80
 
     traces = [
         make_unknown_trace(block_number, first_transaction_hash, []),
@@ -139,11 +150,41 @@ def test_swaps(
             recipient_address=bob_address,
             recipient_input_key="recipient",
         ),
+        make_transfer_trace(
+            block_number,
+            fourth_transaction_hash,
+            trace_address=[2],
+            from_address=danielle_address,
+            to_address=first_converter_address,
+            token_address=fourth_token_in_address,
+            amount=fourth_token_in_amount,
+        ),
+        make_transfer_trace(
+            block_number,
+            fourth_transaction_hash,
+            trace_address=[1, 2],
+            from_address=first_converter_address,
+            to_address=danielle_address,
+            token_address=fourth_token_out_address,
+            amount=fourth_token_out_amount,
+        ),
+        make_swap_trace(
+            block_number,
+            fourth_transaction_hash,
+            trace_address=[],
+            from_address=danielle_address,
+            contract_address=BANCOR_NETWORK_CONTRACT_ADDRESS,
+            abi_name=BANCOR_NETWORK_ABI_NAME,
+            protocol=Protocol.bancor,
+            function_signature="convertByPath(address[],uint256,uint256,address,address,uint256)",
+            recipient_address=danielle_address,
+            recipient_input_key="recipient",
+        ),
     ]
 
     swaps = get_swaps(traces)
 
-    assert len(swaps) == 3
+    assert len(swaps) == 4
 
     for swap in swaps:
         if swap.abi_name == UNISWAP_V2_PAIR_ABI_NAME:
@@ -152,6 +193,8 @@ def test_swaps(
             uni_v3_swap = swap
         elif swap.abi_name == BALANCER_V1_POOL_ABI_NAME:
             bal_v1_swap = swap
+        elif swap.abi_name == BANCOR_NETWORK_ABI_NAME:
+            bancor_swap = swap
         else:
             assert False
 
@@ -193,3 +236,16 @@ def test_swaps(
     assert bal_v1_swap.token_in_amount == third_token_in_amount
     assert bal_v1_swap.token_out_address == third_token_out_address
     assert bal_v1_swap.token_out_amount == third_token_out_amount
+
+    assert bancor_swap.abi_name == BANCOR_NETWORK_ABI_NAME
+    assert bancor_swap.transaction_hash == fourth_transaction_hash
+    assert bancor_swap.block_number == block_number
+    assert bancor_swap.trace_address == []
+    assert bancor_swap.protocol == Protocol.bancor
+    assert bancor_swap.contract_address == BANCOR_NETWORK_CONTRACT_ADDRESS
+    assert bancor_swap.from_address == danielle_address
+    assert bancor_swap.to_address == danielle_address
+    assert bancor_swap.token_in_address == fourth_token_in_address
+    assert bancor_swap.token_in_amount == fourth_token_in_amount
+    assert bancor_swap.token_out_address == fourth_token_out_address
+    assert bancor_swap.token_out_amount == fourth_token_out_amount