Merge remote-tracking branch 'upstream/main' into asyncpg

This commit is contained in:
carlomazzaferro 2021-12-13 15:18:07 -03:00
commit 5591df584d
No known key found for this signature in database
GPG Key ID: 0CED3103EF7B2187
39 changed files with 658 additions and 94 deletions

View File

@ -18,4 +18,5 @@ COPY . /app
# easter eggs 😝
RUN echo "PS1='🕵️:\[\033[1;36m\]\h \[\033[1;34m\]\W\[\033[0;35m\]\[\033[1;36m\]$ \[\033[0m\]'" >> ~/.bashrc
ENTRYPOINT [ "/app/entrypoint.sh"]
ENTRYPOINT [ "poetry" ]
CMD [ "run", "python", "loop.py" ]

View File

@ -1,5 +1,4 @@
load("ext://helm_remote", "helm_remote")
load("ext://restart_process", "docker_build_with_restart")
load("ext://secret", "secret_from_dict")
load("ext://configmap", "configmap_from_dict")
@ -30,8 +29,7 @@ k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = {
# "host": "trace-db-postgresql",
# }))
docker_build_with_restart("mev-inspect-py", ".",
entrypoint="/app/entrypoint.sh",
docker_build("mev-inspect-py", ".",
live_update=[
sync(".", "/app"),
run("cd /app && poetry install",
@ -41,6 +39,10 @@ docker_build_with_restart("mev-inspect-py", ".",
k8s_yaml(helm('./k8s/mev-inspect', name='mev-inspect'))
k8s_resource(workload="mev-inspect", resource_deps=["postgresql-postgresql"])
# uncomment to enable price monitor
# k8s_yaml(helm('./k8s/mev-inspect-prices', name='mev-inspect-prices'))
# k8s_resource(workload="mev-inspect-prices", resource_deps=["postgresql-postgresql"])
local_resource(
'pg-port-forward',
serve_cmd='kubectl port-forward --namespace default svc/postgresql 5432:5432',

View File

@ -0,0 +1,35 @@
"""empty message
Revision ID: 070819d86587
Revises: d498bdb0a641
Create Date: 2021-11-26 18:25:13.402822
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "d498bdb0a641"
down_revision = "205ce02374b3"
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
"punk_snipes",
sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
sa.Column("block_number", sa.Numeric, nullable=False),
sa.Column("transaction_hash", sa.String(66), nullable=False),
sa.Column("trace_address", sa.String(256), nullable=False),
sa.Column("from_address", sa.String(256), nullable=False),
sa.Column("punk_index", sa.Numeric, nullable=False),
sa.Column("min_acceptance_price", sa.Numeric, nullable=False),
sa.Column("acceptance_price", sa.Numeric, nullable=False),
sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
)
def downgrade():
op.drop_table("punk_snipes")

View File

@ -0,0 +1,34 @@
"""empty message
Revision ID: 52d75a7e0533
Revises: 7cf0eeb41da0
Create Date: 2021-11-26 20:35:58.954138
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "52d75a7e0533"
down_revision = "7cf0eeb41da0"
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
"punk_bid_acceptances",
sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
sa.Column("block_number", sa.Numeric, nullable=False),
sa.Column("transaction_hash", sa.String(66), nullable=False),
sa.Column("trace_address", sa.String(256), nullable=False),
sa.Column("from_address", sa.String(256), nullable=False),
sa.Column("punk_index", sa.Numeric, nullable=False),
sa.Column("min_price", sa.Numeric, nullable=False),
sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
)
def downgrade():
op.drop_table("punk_bid_acceptances")

View File

@ -0,0 +1,34 @@
"""empty message
Revision ID: 7cf0eeb41da0
Revises: d498bdb0a641
Create Date: 2021-11-26 20:27:28.936516
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "7cf0eeb41da0"
down_revision = "d498bdb0a641"
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
"punk_bids",
sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
sa.Column("block_number", sa.Numeric, nullable=False),
sa.Column("transaction_hash", sa.String(66), nullable=False),
sa.Column("trace_address", sa.String(256), nullable=False),
sa.Column("from_address", sa.String(256), nullable=False),
sa.Column("punk_index", sa.Numeric, nullable=False),
sa.Column("price", sa.Numeric, nullable=False),
sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
)
def downgrade():
op.drop_table("punk_bids")

View File

@ -0,0 +1,27 @@
"""Remove collateral_token_address column
Revision ID: b9fa1ecc9929
Revises: 04b76ab1d2af
Create Date: 2021-12-01 23:32:40.574108
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "b9fa1ecc9929"
down_revision = "04b76ab1d2af"
branch_labels = None
depends_on = None
def upgrade():
op.drop_column("liquidations", "collateral_token_address")
def downgrade():
op.add_column(
"liquidations",
sa.Column("collateral_token_address", sa.String(256), nullable=False),
)

View File

@ -1,3 +0,0 @@
#!/bin/bash
python loop.py

View File

@ -21,8 +21,7 @@ spec:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
command:
- poetry
args:
- run
- inspect-many-blocks
- {{ .Values.command.startBlockNumber | quote }}

View File

@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

View File

@ -0,0 +1,24 @@
apiVersion: v2
name: mev-inspect-prices
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "1.16.0"

View File

@ -0,0 +1,62 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "mev-inspect-prices.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "mev-inspect-prices.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "mev-inspect-prices.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "mev-inspect-prices.labels" -}}
helm.sh/chart: {{ include "mev-inspect-prices.chart" . }}
{{ include "mev-inspect-prices.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "mev-inspect-prices.selectorLabels" -}}
app.kubernetes.io/name: {{ include "mev-inspect-prices.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "mev-inspect-prices.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "mev-inspect-prices.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}

View File

@ -0,0 +1,35 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: {{ include "mev-inspect-prices.fullname" . }}
spec:
schedule: "0 */1 * * *"
successfulJobsHistoryLimit: 0
jobTemplate:
spec:
template:
spec:
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.image.repository }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
args:
- run
- fetch-all-prices
env:
- name: POSTGRES_HOST
valueFrom:
secretKeyRef:
name: mev-inspect-db-credentials
key: host
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: mev-inspect-db-credentials
key: username
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: mev-inspect-db-credentials
key: password
restartPolicy: Never

View File

@ -0,0 +1,7 @@
image:
repository: mev-inspect-py
pullPolicy: IfNotPresent
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""

View File

@ -30,6 +30,7 @@ spec:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
args: ["run", "python", "loop.py"]
livenessProbe:
exec:
command:

View File

@ -66,7 +66,6 @@ def get_aave_liquidations(
liquidations.append(
Liquidation(
liquidated_user=trace.inputs["_user"],
collateral_token_address=trace.inputs["_collateral"],
debt_token_address=trace.inputs["_reserve"],
liquidator_user=liquidator,
debt_purchase_amount=trace.inputs["_purchaseAmount"],

File diff suppressed because one or more lines are too long

View File

@ -88,8 +88,9 @@ def _get_all_start_end_swaps(swaps: List[Swap]) -> List[Tuple[Swap, Swap]]:
"""
pool_addrs = [swap.contract_address for swap in swaps]
valid_start_ends: List[Tuple[Swap, Swap]] = []
for potential_start_swap in swaps:
for potential_end_swap in swaps:
for index, potential_start_swap in enumerate(swaps):
remaining_swaps = swaps[:index] + swaps[index + 1 :]
for potential_end_swap in remaining_swaps:
if (
potential_start_swap.token_in_address
== potential_end_swap.token_out_address

View File

@ -6,7 +6,7 @@ from mev_inspect.schemas.transfers import Transfer, ETH_TOKEN_ADDRESS
from mev_inspect.schemas.traces import DecodedCallTrace, ClassifiedTrace
def create_swap_from_transfers(
def create_swap_from_pool_transfers(
trace: DecodedCallTrace,
recipient_address: str,
prior_transfers: List[Transfer],
@ -55,6 +55,43 @@ def create_swap_from_transfers(
)
def create_swap_from_recipient_transfers(
trace: DecodedCallTrace,
pool_address: str,
recipient_address: str,
prior_transfers: List[Transfer],
child_transfers: List[Transfer],
) -> Optional[Swap]:
transfers_from_recipient = _filter_transfers(
[*prior_transfers, *child_transfers], from_address=recipient_address
)
transfers_to_recipient = _filter_transfers(
child_transfers, to_address=recipient_address
)
if len(transfers_from_recipient) != 1 or len(transfers_to_recipient) != 1:
return None
transfer_in = transfers_from_recipient[0]
transfer_out = transfers_to_recipient[0]
return Swap(
abi_name=trace.abi_name,
transaction_hash=trace.transaction_hash,
block_number=trace.block_number,
trace_address=trace.trace_address,
contract_address=pool_address,
protocol=trace.protocol,
from_address=transfer_in.from_address,
to_address=transfer_out.to_address,
token_in_address=transfer_in.token_address,
token_in_amount=transfer_in.amount,
token_out_address=transfer_out.token_address,
token_out_amount=transfer_out.amount,
error=trace.error,
)
def _build_eth_transfer(trace: ClassifiedTrace) -> Transfer:
return Transfer(
block_number=trace.block_number,

View File

@ -12,6 +12,7 @@ from .zero_ex import ZEROX_CLASSIFIER_SPECS
from .balancer import BALANCER_CLASSIFIER_SPECS
from .compound import COMPOUND_CLASSIFIER_SPECS
from .cryptopunks import CRYPTOPUNKS_CLASSIFIER_SPECS
from .bancor import BANCOR_CLASSIFIER_SPECS
ALL_CLASSIFIER_SPECS = (
ERC20_CLASSIFIER_SPECS
@ -23,6 +24,7 @@ ALL_CLASSIFIER_SPECS = (
+ BALANCER_CLASSIFIER_SPECS
+ COMPOUND_CLASSIFIER_SPECS
+ CRYPTOPUNKS_CLASSIFIER_SPECS
+ BANCOR_CLASSIFIER_SPECS
)
_SPECS_BY_ABI_NAME_AND_PROTOCOL: Dict[

View File

@ -9,7 +9,7 @@ from mev_inspect.schemas.classifiers import (
ClassifierSpec,
SwapClassifier,
)
from mev_inspect.classifiers.helpers import create_swap_from_transfers
from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
BALANCER_V1_POOL_ABI_NAME = "BPool"
@ -24,7 +24,7 @@ class BalancerSwapClassifier(SwapClassifier):
recipient_address = trace.from_address
swap = create_swap_from_transfers(
swap = create_swap_from_pool_transfers(
trace, recipient_address, prior_transfers, child_transfers
)
return swap

View File

@ -0,0 +1,48 @@
from typing import Optional, List
from mev_inspect.schemas.transfers import Transfer
from mev_inspect.schemas.swaps import Swap
from mev_inspect.schemas.traces import (
DecodedCallTrace,
Protocol,
)
from mev_inspect.schemas.classifiers import (
ClassifierSpec,
SwapClassifier,
)
from mev_inspect.classifiers.helpers import (
create_swap_from_recipient_transfers,
)
BANCOR_NETWORK_ABI_NAME = "BancorNetwork"
BANCOR_NETWORK_CONTRACT_ADDRESS = "0x2F9EC37d6CcFFf1caB21733BdaDEdE11c823cCB0"
class BancorSwapClassifier(SwapClassifier):
@staticmethod
def parse_swap(
trace: DecodedCallTrace,
prior_transfers: List[Transfer],
child_transfers: List[Transfer],
) -> Optional[Swap]:
recipient_address = trace.from_address
swap = create_swap_from_recipient_transfers(
trace,
BANCOR_NETWORK_CONTRACT_ADDRESS,
recipient_address,
prior_transfers,
child_transfers,
)
return swap
BANCOR_NETWORK_SPEC = ClassifierSpec(
abi_name=BANCOR_NETWORK_ABI_NAME,
protocol=Protocol.bancor,
classifiers={
"convertByPath(address[],uint256,uint256,address,address,uint256)": BancorSwapClassifier,
},
valid_contract_addresses=[BANCOR_NETWORK_CONTRACT_ADDRESS],
)
BANCOR_CLASSIFIER_SPECS = [BANCOR_NETWORK_SPEC]

View File

@ -10,7 +10,7 @@ from mev_inspect.schemas.classifiers import (
ClassifierSpec,
SwapClassifier,
)
from mev_inspect.classifiers.helpers import create_swap_from_transfers
from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
class CurveSwapClassifier(SwapClassifier):
@ -23,7 +23,7 @@ class CurveSwapClassifier(SwapClassifier):
recipient_address = trace.from_address
swap = create_swap_from_transfers(
swap = create_swap_from_pool_transfers(
trace, recipient_address, prior_transfers, child_transfers
)
return swap

View File

@ -9,7 +9,7 @@ from mev_inspect.schemas.classifiers import (
ClassifierSpec,
SwapClassifier,
)
from mev_inspect.classifiers.helpers import create_swap_from_transfers
from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
UNISWAP_V2_PAIR_ABI_NAME = "UniswapV2Pair"
@ -26,7 +26,7 @@ class UniswapV3SwapClassifier(SwapClassifier):
recipient_address = trace.inputs.get("recipient", trace.from_address)
swap = create_swap_from_transfers(
swap = create_swap_from_pool_transfers(
trace, recipient_address, prior_transfers, child_transfers
)
return swap
@ -42,7 +42,7 @@ class UniswapV2SwapClassifier(SwapClassifier):
recipient_address = trace.inputs.get("to", trace.from_address)
swap = create_swap_from_transfers(
swap = create_swap_from_pool_transfers(
trace, recipient_address, prior_transfers, child_transfers
)
return swap

View File

@ -3,12 +3,27 @@ import aiohttp
from mev_inspect.classifiers.specs.weth import WETH_ADDRESS
from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS
from mev_inspect.schemas.coinbase import CoinbasePrices, CoinbasePricesResponse
from mev_inspect.schemas.prices import (
WBTC_TOKEN_ADDRESS,
LINK_TOKEN_ADDRESS,
YEARN_TOKEN_ADDRESS,
AAVE_TOKEN_ADDRESS,
UNI_TOKEN_ADDRESS,
USDC_TOKEN_ADDRESS_ADDRESS,
REN_TOKEN_ADDRESS,
)
COINBASE_API_BASE = "https://www.coinbase.com/api/v2"
COINBASE_TOKEN_NAME_BY_ADDRESS = {
WETH_ADDRESS: "weth",
ETH_TOKEN_ADDRESS: "ethereum",
WBTC_TOKEN_ADDRESS: "wrapped-bitcoin",
LINK_TOKEN_ADDRESS: "link",
YEARN_TOKEN_ADDRESS: "yearn-finance",
AAVE_TOKEN_ADDRESS: "aave",
UNI_TOKEN_ADDRESS: "uniswap",
USDC_TOKEN_ADDRESS_ADDRESS: "usdc",
REN_TOKEN_ADDRESS: "ren",
}

View File

@ -1,5 +1,4 @@
from typing import Dict, List, Optional
from web3 import Web3
from typing import List, Optional
from mev_inspect.traces import get_child_traces
from mev_inspect.schemas.traces import (
@ -9,44 +8,15 @@ from mev_inspect.schemas.traces import (
)
from mev_inspect.schemas.liquidations import Liquidation
from mev_inspect.abi import get_raw_abi
from mev_inspect.transfers import ETH_TOKEN_ADDRESS
V2_COMPTROLLER_ADDRESS = "0x3d9819210A31b4961b30EF54bE2aeD79B9c9Cd3B"
V2_C_ETHER = "0x4Ddc2D193948926D02f9B1fE9e1daa0718270ED5"
CREAM_COMPTROLLER_ADDRESS = "0x3d5BC3c8d13dcB8bF317092d84783c2697AE9258"
CREAM_CR_ETHER = "0xD06527D5e56A3495252A528C4987003b712860eE"
# helper, only queried once in the beginning (inspect_block)
def fetch_all_underlying_markets(w3: Web3, protocol: Protocol) -> Dict[str, str]:
if protocol == Protocol.compound_v2:
c_ether = V2_C_ETHER
address = V2_COMPTROLLER_ADDRESS
elif protocol == Protocol.cream:
c_ether = CREAM_CR_ETHER
address = CREAM_COMPTROLLER_ADDRESS
else:
raise ValueError(f"No Comptroller found for {protocol}")
token_mapping = {}
comptroller_abi = get_raw_abi("Comptroller", Protocol.compound_v2)
comptroller_instance = w3.eth.contract(address=address, abi=comptroller_abi)
markets = comptroller_instance.functions.getAllMarkets().call()
token_abi = get_raw_abi("CToken", Protocol.compound_v2)
for token in markets:
# make an exception for cETH (as it has no .underlying())
if token != c_ether:
token_instance = w3.eth.contract(address=token, abi=token_abi)
underlying_token = token_instance.functions.underlying().call()
token_mapping[
token.lower()
] = underlying_token.lower() # make k:v lowercase for consistancy
return token_mapping
def get_compound_liquidations(
traces: List[ClassifiedTrace],
collateral_by_c_token_address: Dict[str, str],
collateral_by_cr_token_address: Dict[str, str],
) -> List[Liquidation]:
"""Inspect list of classified traces and identify liquidation"""
@ -67,23 +37,13 @@ def get_compound_liquidations(
trace.transaction_hash, trace.trace_address, traces
)
seize_trace = _get_seize_call(child_traces)
underlying_markets = {}
if trace.protocol == Protocol.compound_v2:
underlying_markets = collateral_by_c_token_address
elif trace.protocol == Protocol.cream:
underlying_markets = collateral_by_cr_token_address
if (
seize_trace is not None
and seize_trace.inputs is not None
and len(underlying_markets) != 0
):
if seize_trace is not None and seize_trace.inputs is not None:
c_token_collateral = trace.inputs["cTokenCollateral"]
if trace.abi_name == "CEther":
liquidations.append(
Liquidation(
liquidated_user=trace.inputs["borrower"],
collateral_token_address=ETH_TOKEN_ADDRESS, # WETH since all cEther liquidations provide Ether
debt_token_address=c_token_collateral,
liquidator_user=seize_trace.inputs["liquidator"],
debt_purchase_amount=trace.value,
@ -97,13 +57,9 @@ def get_compound_liquidations(
elif (
trace.abi_name == "CToken"
): # cToken liquidations where liquidator pays back via token transfer
c_token_address = trace.to_address
liquidations.append(
Liquidation(
liquidated_user=trace.inputs["borrower"],
collateral_token_address=underlying_markets[
c_token_address
],
debt_token_address=c_token_collateral,
liquidator_user=seize_trace.inputs["liquidator"],
debt_purchase_amount=trace.inputs["repayAmount"],

View File

@ -1,3 +1,5 @@
from datetime import datetime
from mev_inspect.schemas.blocks import Block
@ -21,7 +23,7 @@ async def write_block(
"INSERT INTO blocks (block_number, block_timestamp) VALUES (:block_number, :block_timestamp)",
params={
"block_number": block.block_number,
"block_timestamp": block.block_timestamp,
"block_timestamp": datetime.fromtimestamp(block.block_timestamp),
},
)
await db_session.commit()

85
mev_inspect/crud/punks.py Normal file
View File

@ -0,0 +1,85 @@
import json
from typing import List
from mev_inspect.models.punks import (
PunkSnipeModel,
PunkBidModel,
PunkBidAcceptanceModel,
)
from mev_inspect.schemas.punk_snipe import PunkSnipe
from mev_inspect.schemas.punk_bid import PunkBid
from mev_inspect.schemas.punk_accept_bid import PunkBidAcceptance
def delete_punk_bid_acceptances_for_block(
db_session,
block_number: int,
) -> None:
(
db_session.query(PunkBidAcceptanceModel)
.filter(PunkBidAcceptanceModel.block_number == block_number)
.delete()
)
db_session.commit()
def write_punk_bid_acceptances(
db_session,
punk_bid_acceptances: List[PunkBidAcceptance],
) -> None:
models = [
PunkBidAcceptanceModel(**json.loads(punk_bid_acceptance.json()))
for punk_bid_acceptance in punk_bid_acceptances
]
db_session.bulk_save_objects(models)
db_session.commit()
def delete_punk_bids_for_block(
db_session,
block_number: int,
) -> None:
(
db_session.query(PunkBidModel)
.filter(PunkBidModel.block_number == block_number)
.delete()
)
db_session.commit()
def write_punk_bids(
db_session,
punk_bids: List[PunkBid],
) -> None:
models = [PunkBidModel(**json.loads(punk_bid.json())) for punk_bid in punk_bids]
db_session.bulk_save_objects(models)
db_session.commit()
def delete_punk_snipes_for_block(
db_session,
block_number: int,
) -> None:
(
db_session.query(PunkSnipeModel)
.filter(PunkSnipeModel.block_number == block_number)
.delete()
)
db_session.commit()
def write_punk_snipes(
db_session,
punk_snipes: List[PunkSnipe],
) -> None:
models = [
PunkSnipeModel(**json.loads(punk_snipe.json())) for punk_snipe in punk_snipes
]
db_session.bulk_save_objects(models)
db_session.commit()

View File

@ -11,6 +11,16 @@ from mev_inspect.crud.arbitrages import (
delete_arbitrages_for_block,
write_arbitrages,
)
from mev_inspect.crud.punks import (
delete_punk_snipes_for_block,
write_punk_snipes,
delete_punk_bids_for_block,
write_punk_bids,
delete_punk_bid_acceptances_for_block,
write_punk_bid_acceptances,
)
from mev_inspect.crud.blocks import (
delete_block,
write_block,
@ -92,11 +102,19 @@ async def inspect_block(
await write_liquidations(inspect_db_session, liquidations)
punk_bids = get_punk_bids(classified_traces)
delete_punk_bids_for_block(inspect_db_session, block_number)
write_punk_bids(inspect_db_session, punk_bids)
punk_bid_acceptances = get_punk_bid_acceptances(classified_traces)
delete_punk_bid_acceptances_for_block(inspect_db_session, block_number)
write_punk_bid_acceptances(inspect_db_session, punk_bid_acceptances)
punk_snipes = get_punk_snipes(punk_bids, punk_bid_acceptances)
logger.info(f"Block: {block_number} -- Found {len(punk_snipes)} punk snipes")
delete_punk_snipes_for_block(inspect_db_session, block_number)
write_punk_snipes(inspect_db_session, punk_snipes)
miner_payments = get_miner_payments(
block.miner, block.base_fee_per_gas, classified_traces, block.receipts
)

View File

@ -1,6 +1,7 @@
from typing import List
from mev_inspect.aave_liquidations import get_aave_liquidations
from mev_inspect.compound_liquidations import get_compound_liquidations
from mev_inspect.schemas.traces import (
ClassifiedTrace,
Classification,
@ -20,4 +21,5 @@ def get_liquidations(
classified_traces: List[ClassifiedTrace],
) -> List[Liquidation]:
aave_liquidations = get_aave_liquidations(classified_traces)
return aave_liquidations
comp_liquidations = get_compound_liquidations(classified_traces)
return aave_liquidations + comp_liquidations

View File

@ -8,7 +8,6 @@ class LiquidationModel(Base):
liquidated_user = Column(String, nullable=False)
liquidator_user = Column(String, nullable=False)
collateral_token_address = Column(String, nullable=False)
debt_token_address = Column(String, nullable=False)
debt_purchase_amount = Column(Numeric, nullable=False)
received_amount = Column(Numeric, nullable=False)

View File

@ -0,0 +1,15 @@
from sqlalchemy import Column, Numeric, String, ARRAY, Integer
from .base import Base
class PunkSnipeModel(Base):
__tablename__ = "punk_snipes"
block_number = Column(Numeric, nullable=False)
transaction_hash = Column(String, primary_key=True)
trace_address = Column(ARRAY(Integer), primary_key=True)
from_address = Column(String, nullable=False)
punk_index = Column(Integer, nullable=False)
min_acceptance_price = Column(Numeric, nullable=False)
acceptance_price = Column(Numeric, nullable=False)

View File

@ -0,0 +1,37 @@
from sqlalchemy import Column, Numeric, String, ARRAY, Integer
from .base import Base
class PunkSnipeModel(Base):
__tablename__ = "punk_snipes"
block_number = Column(Numeric, nullable=False)
transaction_hash = Column(String, primary_key=True)
trace_address = Column(ARRAY(Integer), primary_key=True)
from_address = Column(String, nullable=False)
punk_index = Column(Integer, nullable=False)
min_acceptance_price = Column(Numeric, nullable=False)
acceptance_price = Column(Numeric, nullable=False)
class PunkBidModel(Base):
__tablename__ = "punk_bids"
block_number = Column(Numeric, nullable=False)
transaction_hash = Column(String, primary_key=True)
trace_address = Column(ARRAY(Integer), primary_key=True)
from_address = Column(String, nullable=False)
punk_index = Column(Integer, nullable=False)
price = Column(Numeric, nullable=False)
class PunkBidAcceptanceModel(Base):
__tablename__ = "punk_bid_acceptances"
block_number = Column(Numeric, nullable=False)
transaction_hash = Column(String, primary_key=True)
trace_address = Column(ARRAY(Integer), primary_key=True)
from_address = Column(String, nullable=False)
punk_index = Column(Integer, nullable=False)
min_price = Column(Numeric, nullable=False)

View File

@ -2,13 +2,29 @@ from typing import List
from mev_inspect.classifiers.specs.weth import WETH_ADDRESS
from mev_inspect.coinbase import fetch_coinbase_prices
from mev_inspect.schemas.prices import Price
from mev_inspect.schemas.prices import (
Price,
WBTC_TOKEN_ADDRESS,
LINK_TOKEN_ADDRESS,
YEARN_TOKEN_ADDRESS,
AAVE_TOKEN_ADDRESS,
UNI_TOKEN_ADDRESS,
USDC_TOKEN_ADDRESS_ADDRESS,
REN_TOKEN_ADDRESS,
)
from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS
SUPPORTED_TOKENS = [
WETH_ADDRESS,
ETH_TOKEN_ADDRESS,
LINK_TOKEN_ADDRESS,
AAVE_TOKEN_ADDRESS,
USDC_TOKEN_ADDRESS_ADDRESS,
REN_TOKEN_ADDRESS,
WBTC_TOKEN_ADDRESS,
YEARN_TOKEN_ADDRESS,
UNI_TOKEN_ADDRESS,
]

View File

@ -6,7 +6,6 @@ from mev_inspect.schemas.traces import Protocol
class Liquidation(BaseModel):
liquidated_user: str
liquidator_user: str
collateral_token_address: str
debt_token_address: str
debt_purchase_amount: int
received_amount: int

View File

@ -2,6 +2,14 @@ from datetime import datetime
from pydantic import BaseModel
WBTC_TOKEN_ADDRESS = "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599"
LINK_TOKEN_ADDRESS = "0x514910771af9ca656af840dff83e8264ecf986ca"
YEARN_TOKEN_ADDRESS = "0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e"
AAVE_TOKEN_ADDRESS = "0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9"
UNI_TOKEN_ADDRESS = "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984"
USDC_TOKEN_ADDRESS_ADDRESS = "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"
REN_TOKEN_ADDRESS = "0x408e41876cccdc0f92210600ef50372656052a38"
class Price(BaseModel):
token_address: str

View File

@ -47,6 +47,7 @@ class Protocol(Enum):
compound_v2 = "compound_v2"
cream = "cream"
cryptopunks = "cryptopunks"
bancor = "bancor"
class ClassifiedTrace(Trace):

View File

@ -19,7 +19,6 @@ def test_single_weth_liquidation():
Liquidation(
liquidated_user="0xd16404ca0a74a15e66d8ad7c925592fb02422ffe",
liquidator_user="0x19256c009781bc2d1545db745af6dfd30c7e9cfa",
collateral_token_address="0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2",
debt_token_address="0xdac17f958d2ee523a2206206994597c13d831ec7",
debt_purchase_amount=26503300291,
received_amount=8182733924513576561,
@ -50,7 +49,6 @@ def test_single_liquidation():
Liquidation(
liquidated_user="0x8d8d912fe4db5917da92d14fea05225b803c359c",
liquidator_user="0xf2d9e54f0e317b8ac94825b2543908e7552fe9c7",
collateral_token_address="0x80fb784b7ed66730e8b1dbd9820afd29931aab03",
debt_token_address="0xdac17f958d2ee523a2206206994597c13d831ec7",
debt_purchase_amount=1069206535,
received_amount=2657946947610159065393,
@ -81,7 +79,6 @@ def test_single_liquidation_with_atoken_payback():
Liquidation(
liquidated_user="0x3d2b6eacd1bca51af57ed8b3ff9ef0bd8ee8c56d",
liquidator_user="0x887668f2dc9612280243f2a6ef834cecf456654e",
collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
debt_token_address="0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2",
debt_purchase_amount=767615458043667978,
received_amount=113993647930952952550,
@ -111,7 +108,6 @@ def test_multiple_liquidations_in_block():
liquidation1 = Liquidation(
liquidated_user="0x6c6541ae8a7c6a6f968124a5ff2feac8f0c7875b",
liquidator_user="0x7185e240d8e9e2d692cbc68d30eecf965e9a7feb",
collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
debt_token_address="0x4fabb145d64652a948d72533023f6e7a623c7c53",
debt_purchase_amount=457700000000000000000,
received_amount=10111753901939162887,
@ -125,7 +121,6 @@ def test_multiple_liquidations_in_block():
liquidation2 = Liquidation(
liquidated_user="0x6c6541ae8a7c6a6f968124a5ff2feac8f0c7875b",
liquidator_user="0x7185e240d8e9e2d692cbc68d30eecf965e9a7feb",
collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
debt_token_address="0x0000000000085d4780b73119b644ae5ecd22b376",
debt_purchase_amount=497030000000000000000,
received_amount=21996356316098208090,
@ -139,7 +134,6 @@ def test_multiple_liquidations_in_block():
liquidation3 = Liquidation(
liquidated_user="0xda874f844389df33c0fad140df4970fe1b366726",
liquidator_user="0x7185e240d8e9e2d692cbc68d30eecf965e9a7feb",
collateral_token_address="0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2",
debt_token_address="0x57ab1ec28d129707052df4df418d58a2d46d5f51",
debt_purchase_amount=447810000000000000000,
received_amount=121531358145247546,
@ -169,7 +163,6 @@ def test_liquidations_with_eth_transfer():
liquidation1 = Liquidation(
liquidated_user="0xad346c7762f74c78da86d2941c6eb546e316fbd0",
liquidator_user="0x27239549dd40e1d60f5b80b0c4196923745b1fd2",
collateral_token_address=ETH_TOKEN_ADDRESS,
debt_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
debt_purchase_amount=1809152000000000000,
received_amount=15636807387264000,
@ -183,7 +176,6 @@ def test_liquidations_with_eth_transfer():
liquidation2 = Liquidation(
liquidated_user="0xad346c7762f74c78da86d2941c6eb546e316fbd0",
liquidator_user="0x27239549dd40e1d60f5b80b0c4196923745b1fd2",
collateral_token_address=ETH_TOKEN_ADDRESS,
debt_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
debt_purchase_amount=1809152000000000000,
received_amount=8995273139160873,

View File

@ -2,7 +2,6 @@ from mev_inspect.compound_liquidations import get_compound_liquidations
from mev_inspect.schemas.liquidations import Liquidation
from mev_inspect.schemas.traces import Protocol
from mev_inspect.classifiers.trace import TraceClassifier
from mev_inspect.transfers import ETH_TOKEN_ADDRESS
from tests.utils import load_test_block, load_comp_markets, load_cream_markets
comp_markets = load_comp_markets()
@ -19,7 +18,6 @@ def test_c_ether_liquidations():
Liquidation(
liquidated_user="0xb5535a3681cf8d5431b8acfd779e2f79677ecce9",
liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
collateral_token_address=ETH_TOKEN_ADDRESS,
debt_token_address="0x39aa39c021dfbae8fac545936693ac917d5e7563",
debt_purchase_amount=268066492249420078,
received_amount=4747650169097,
@ -32,7 +30,7 @@ def test_c_ether_liquidations():
block = load_test_block(block_number)
trace_classifier = TraceClassifier()
classified_traces = trace_classifier.classify(block.traces)
result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
result = get_compound_liquidations(classified_traces)
assert result == liquidations
block_number = 13207907
@ -44,7 +42,6 @@ def test_c_ether_liquidations():
Liquidation(
liquidated_user="0x45df6f00166c3fb77dc16b9e47ff57bc6694e898",
liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
collateral_token_address=ETH_TOKEN_ADDRESS,
debt_token_address="0x35a18000230da775cac24873d00ff85bccded550",
debt_purchase_amount=414547860568297082,
received_amount=321973320649,
@ -58,7 +55,7 @@ def test_c_ether_liquidations():
block = load_test_block(block_number)
trace_classifier = TraceClassifier()
classified_traces = trace_classifier.classify(block.traces)
result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
result = get_compound_liquidations(classified_traces)
assert result == liquidations
block_number = 13298725
@ -70,7 +67,6 @@ def test_c_ether_liquidations():
Liquidation(
liquidated_user="0xacbcf5d2970eef25f02a27e9d9cd31027b058b9b",
liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
collateral_token_address=ETH_TOKEN_ADDRESS,
debt_token_address="0x35a18000230da775cac24873d00ff85bccded550",
debt_purchase_amount=1106497772527562662,
received_amount=910895850496,
@ -83,7 +79,7 @@ def test_c_ether_liquidations():
block = load_test_block(block_number)
trace_classifier = TraceClassifier()
classified_traces = trace_classifier.classify(block.traces)
result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
result = get_compound_liquidations(classified_traces)
assert result == liquidations
@ -97,7 +93,6 @@ def test_c_token_liquidation():
Liquidation(
liquidated_user="0xacdd5528c1c92b57045041b5278efa06cdade4d8",
liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
collateral_token_address="0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48",
debt_token_address="0x70e36f6bf80a52b3b46b3af8e106cc0ed743e8e4",
debt_purchase_amount=1207055531,
received_amount=21459623305,
@ -110,7 +105,7 @@ def test_c_token_liquidation():
block = load_test_block(block_number)
trace_classifier = TraceClassifier()
classified_traces = trace_classifier.classify(block.traces)
result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
result = get_compound_liquidations(classified_traces)
assert result == liquidations
@ -124,7 +119,6 @@ def test_cream_token_liquidation():
Liquidation(
liquidated_user="0x46bf9479dc569bc796b7050344845f6564d45fba",
liquidator_user="0xa2863cad9c318669660eb4eca8b3154b90fb4357",
collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
debt_token_address="0x44fbebd2f576670a6c33f6fc0b00aa8c5753b322",
debt_purchase_amount=14857434973806369550,
received_amount=1547215810826,
@ -137,5 +131,5 @@ def test_cream_token_liquidation():
block = load_test_block(block_number)
trace_classifier = TraceClassifier()
classified_traces = trace_classifier.classify(block.traces)
result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
result = get_compound_liquidations(classified_traces)
assert result == liquidations

View File

@ -4,6 +4,10 @@ from mev_inspect.classifiers.specs.uniswap import (
UNISWAP_V2_PAIR_ABI_NAME,
UNISWAP_V3_POOL_ABI_NAME,
)
from mev_inspect.classifiers.specs.bancor import (
BANCOR_NETWORK_ABI_NAME,
BANCOR_NETWORK_CONTRACT_ADDRESS,
)
from mev_inspect.schemas.traces import Protocol
from .helpers import (
@ -23,12 +27,14 @@ def test_swaps(
first_transaction_hash,
second_transaction_hash,
third_transaction_hash,
] = get_transaction_hashes(3)
fourth_transaction_hash,
] = get_transaction_hashes(4)
[
alice_address,
bob_address,
carl_address,
danielle_address,
first_token_in_address,
first_token_out_address,
first_pool_address,
@ -38,7 +44,10 @@ def test_swaps(
third_token_in_address,
third_token_out_address,
third_pool_address,
] = get_addresses(12)
fourth_token_in_address,
fourth_token_out_address,
first_converter_address,
] = get_addresses(16)
first_token_in_amount = 10
first_token_out_amount = 20
@ -46,6 +55,8 @@ def test_swaps(
second_token_out_amount = 40
third_token_in_amount = 50
third_token_out_amount = 60
fourth_token_in_amount = 70
fourth_token_out_amount = 80
traces = [
make_unknown_trace(block_number, first_transaction_hash, []),
@ -139,11 +150,41 @@ def test_swaps(
recipient_address=bob_address,
recipient_input_key="recipient",
),
make_transfer_trace(
block_number,
fourth_transaction_hash,
trace_address=[2],
from_address=danielle_address,
to_address=first_converter_address,
token_address=fourth_token_in_address,
amount=fourth_token_in_amount,
),
make_transfer_trace(
block_number,
fourth_transaction_hash,
trace_address=[1, 2],
from_address=first_converter_address,
to_address=danielle_address,
token_address=fourth_token_out_address,
amount=fourth_token_out_amount,
),
make_swap_trace(
block_number,
fourth_transaction_hash,
trace_address=[],
from_address=danielle_address,
contract_address=BANCOR_NETWORK_CONTRACT_ADDRESS,
abi_name=BANCOR_NETWORK_ABI_NAME,
protocol=Protocol.bancor,
function_signature="convertByPath(address[],uint256,uint256,address,address,uint256)",
recipient_address=danielle_address,
recipient_input_key="recipient",
),
]
swaps = get_swaps(traces)
assert len(swaps) == 3
assert len(swaps) == 4
for swap in swaps:
if swap.abi_name == UNISWAP_V2_PAIR_ABI_NAME:
@ -152,6 +193,8 @@ def test_swaps(
uni_v3_swap = swap
elif swap.abi_name == BALANCER_V1_POOL_ABI_NAME:
bal_v1_swap = swap
elif swap.abi_name == BANCOR_NETWORK_ABI_NAME:
bancor_swap = swap
else:
assert False
@ -193,3 +236,16 @@ def test_swaps(
assert bal_v1_swap.token_in_amount == third_token_in_amount
assert bal_v1_swap.token_out_address == third_token_out_address
assert bal_v1_swap.token_out_amount == third_token_out_amount
assert bancor_swap.abi_name == BANCOR_NETWORK_ABI_NAME
assert bancor_swap.transaction_hash == fourth_transaction_hash
assert bancor_swap.block_number == block_number
assert bancor_swap.trace_address == []
assert bancor_swap.protocol == Protocol.bancor
assert bancor_swap.contract_address == BANCOR_NETWORK_CONTRACT_ADDRESS
assert bancor_swap.from_address == danielle_address
assert bancor_swap.to_address == danielle_address
assert bancor_swap.token_in_address == fourth_token_in_address
assert bancor_swap.token_in_amount == fourth_token_in_amount
assert bancor_swap.token_out_address == fourth_token_out_address
assert bancor_swap.token_out_amount == fourth_token_out_amount