merge commit

This commit is contained in:
carlomazzaferro 2021-11-28 13:19:42 +01:00
commit 3fa19b6cfe
No known key found for this signature in database
GPG Key ID: 0CED3103EF7B2187
26 changed files with 627 additions and 1 deletions

View File

@ -0,0 +1,36 @@
"""Change blocks.timestamp to timestamp
Revision ID: 04b76ab1d2af
Revises: 2c90b2b8a80b
Create Date: 2021-11-26 15:31:21.111693
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "04b76ab1d2af"
down_revision = "0cef835f7b36"
branch_labels = None
depends_on = None
def upgrade():
op.alter_column(
"blocks",
"block_timestamp",
type_=sa.TIMESTAMP,
nullable=False,
postgresql_using="TO_TIMESTAMP(block_timestamp)",
)
def downgrade():
op.alter_column(
"blocks",
"block_timestamp",
type_=sa.Numeric,
nullable=False,
postgresql_using="extract(epoch FROM block_timestamp)",
)

15
cli.py
View File

@ -5,12 +5,15 @@ import sys
import click
from mev_inspect.concurrency import coro
from mev_inspect.crud.prices import write_prices
from mev_inspect.db import get_sessions
from mev_inspect.inspector import MEVInspector
from mev_inspect.prices import fetch_all_supported_prices
RPC_URL_ENV = "RPC_URL"
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logger = logging.getLogger(__name__)
@click.group()
@ -78,6 +81,18 @@ async def inspect_many_blocks_command(
)
@cli.command()
@coro
async def fetch_all_prices():
inspect_session, _ = get_sessions()
logger.info("Fetching prices")
prices = await fetch_all_supported_prices()
logger.info("Writing prices")
write_prices(inspect_session, prices)
def get_rpc_url() -> str:
return os.environ["RPC_URL"]

13
mev
View File

@ -56,6 +56,19 @@ case "$1" in
echo "Fetching block $block_number"
kubectl exec -ti deploy/mev-inspect -- poetry run fetch-block $block_number
;;
prices)
shift
case "$1" in
fetch-all)
echo "Running price fetch-all"
kubectl exec -ti deploy/mev-inspect -- \
poetry run fetch-all-prices
;;
*)
echo "prices usage: "$1" {fetch-all}"
exit 1
esac
;;
exec)
shift
kubectl exec -ti deploy/mev-inspect -- $@

File diff suppressed because one or more lines are too long

View File

@ -11,6 +11,7 @@ from .weth import WETH_CLASSIFIER_SPECS, WETH_ADDRESS
from .zero_ex import ZEROX_CLASSIFIER_SPECS
from .balancer import BALANCER_CLASSIFIER_SPECS
from .compound import COMPOUND_CLASSIFIER_SPECS
from .cryptopunks import CRYPTOPUNKS_CLASSIFIER_SPECS
ALL_CLASSIFIER_SPECS = (
ERC20_CLASSIFIER_SPECS
@ -21,6 +22,7 @@ ALL_CLASSIFIER_SPECS = (
+ ZEROX_CLASSIFIER_SPECS
+ BALANCER_CLASSIFIER_SPECS
+ COMPOUND_CLASSIFIER_SPECS
+ CRYPTOPUNKS_CLASSIFIER_SPECS
)
_SPECS_BY_ABI_NAME_AND_PROTOCOL: Dict[

View File

@ -0,0 +1,31 @@
from mev_inspect.schemas.traces import Protocol, Classification
from mev_inspect.schemas.classifiers import (
ClassifierSpec,
Classifier,
)
class PunkBidAcceptanceClassifier(Classifier):
@staticmethod
def get_classification() -> Classification:
return Classification.punk_accept_bid
class PunkBidClassifier(Classifier):
@staticmethod
def get_classification() -> Classification:
return Classification.punk_bid
CRYPTO_PUNKS_SPEC = ClassifierSpec(
abi_name="cryptopunks",
protocol=Protocol.cryptopunks,
valid_contract_addresses=["0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB"],
classifiers={
"enterBidForPunk(uint256)": PunkBidClassifier,
"acceptBidForPunk(uint256,uint256)": PunkBidAcceptanceClassifier,
},
)
CRYPTOPUNKS_CLASSIFIER_SPECS = [CRYPTO_PUNKS_SPEC]

View File

@ -1,10 +1,59 @@
from typing import Optional, List, Tuple
from mev_inspect.schemas.transfers import Transfer
from mev_inspect.schemas.swaps import Swap
from mev_inspect.schemas.traces import (
DecodedCallTrace,
Protocol,
)
from mev_inspect.schemas.classifiers import (
ClassifierSpec,
SwapClassifier,
)
ANY_TAKER_ADDRESS = "0x0000000000000000000000000000000000000000"
RFQ_SIGNATURES = [
"fillRfqOrder((address,address,uint128,uint128,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128)",
"_fillRfqOrder((address,address,uint128,uint128,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128,address,bool,address)",
]
LIMIT_SIGNATURES = [
"fillOrKillLimitOrder((address,address,uint128,uint128,uint128,address,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128)",
"fillLimitOrder((address,address,uint128,uint128,uint128,address,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128)",
"_fillLimitOrder((address,address,uint128,uint128,uint128,address,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128,address,address)",
]
class ZeroExSwapClassifier(SwapClassifier):
@staticmethod
def parse_swap(
trace: DecodedCallTrace,
prior_transfers: List[Transfer],
child_transfers: List[Transfer],
) -> Optional[Swap]:
token_in_address, token_in_amount = _get_0x_token_in_data(
trace, child_transfers
)
token_out_address, token_out_amount = _get_0x_token_out_data(trace)
return Swap(
abi_name=trace.abi_name,
transaction_hash=trace.transaction_hash,
block_number=trace.block_number,
trace_address=trace.trace_address,
contract_address=trace.to_address,
protocol=Protocol.zero_ex,
from_address=trace.from_address,
to_address=trace.to_address,
token_in_address=token_in_address,
token_in_amount=token_in_amount,
token_out_address=token_out_address,
token_out_amount=token_out_amount,
error=trace.error,
)
ZEROX_CONTRACT_SPECS = [
ClassifierSpec(
abi_name="exchangeProxy",
@ -121,6 +170,14 @@ ZEROX_GENERIC_SPECS = [
ClassifierSpec(
abi_name="INativeOrdersFeature",
protocol=Protocol.zero_ex,
valid_contract_addresses=["0xdef1c0ded9bec7f1a1670819833240f027b25eff"],
classifiers={
"fillOrKillLimitOrder((address,address,uint128,uint128,uint128,address,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128)": ZeroExSwapClassifier,
"fillRfqOrder((address,address,uint128,uint128,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128)": ZeroExSwapClassifier,
"fillLimitOrder((address,address,uint128,uint128,uint128,address,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128)": ZeroExSwapClassifier,
"_fillRfqOrder((address,address,uint128,uint128,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128,address,bool,address)": ZeroExSwapClassifier,
"_fillLimitOrder((address,address,uint128,uint128,uint128,address,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128,address,address)": ZeroExSwapClassifier,
},
),
ClassifierSpec(
abi_name="IOtcOrdersFeature",
@ -165,3 +222,57 @@ ZEROX_GENERIC_SPECS = [
]
ZEROX_CLASSIFIER_SPECS = ZEROX_CONTRACT_SPECS + ZEROX_GENERIC_SPECS
def _get_taker_token_in_amount(
taker_address: str, token_in_address: str, child_transfers: List[Transfer]
) -> int:
if len(child_transfers) != 2:
raise ValueError(
f"A settled order should consist of 2 child transfers, not {len(child_transfers)}."
)
if taker_address == ANY_TAKER_ADDRESS:
for transfer in child_transfers:
if transfer.token_address == token_in_address:
return transfer.amount
else:
for transfer in child_transfers:
if transfer.to_address == taker_address:
return transfer.amount
return 0
def _get_0x_token_in_data(
trace: DecodedCallTrace, child_transfers: List[Transfer]
) -> Tuple[str, int]:
order: List = trace.inputs["order"]
token_in_address = order[0]
if trace.function_signature in RFQ_SIGNATURES:
taker_address = order[5]
elif trace.function_signature in LIMIT_SIGNATURES:
taker_address = order[6]
else:
raise RuntimeError(
f"0x orderbook function {trace.function_signature} is not supported"
)
token_in_amount = _get_taker_token_in_amount(
taker_address, token_in_address, child_transfers
)
return token_in_address, token_in_amount
def _get_0x_token_out_data(trace: DecodedCallTrace) -> Tuple[str, int]:
order: List = trace.inputs["order"]
token_out_address = order[1]
token_out_amount = trace.inputs["takerTokenFillAmount"]
return token_out_address, token_out_amount

25
mev_inspect/coinbase.py Normal file
View File

@ -0,0 +1,25 @@
import aiohttp
from mev_inspect.classifiers.specs.weth import WETH_ADDRESS
from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS
from mev_inspect.schemas.coinbase import CoinbasePrices, CoinbasePricesResponse
COINBASE_API_BASE = "https://www.coinbase.com/api/v2"
COINBASE_TOKEN_NAME_BY_ADDRESS = {
WETH_ADDRESS: "weth",
ETH_TOKEN_ADDRESS: "ethereum",
}
async def fetch_coinbase_prices(token_address: str) -> CoinbasePrices:
if token_address not in COINBASE_TOKEN_NAME_BY_ADDRESS:
raise ValueError(f"Unsupported token_address {token_address}")
coinbase_token_name = COINBASE_TOKEN_NAME_BY_ADDRESS[token_address]
url = f"{COINBASE_API_BASE}/assets/prices/{coinbase_token_name}"
async with aiohttp.ClientSession() as session:
async with session.get(url, params={"base": "USD"}) as response:
json_data = await response.json()
return CoinbasePricesResponse(**json_data).data.prices

View File

@ -0,0 +1,17 @@
from typing import List
from sqlalchemy.dialects.postgresql import insert
from mev_inspect.models.prices import PriceModel
from mev_inspect.schemas.prices import Price
def write_prices(db_session, prices: List[Price]) -> None:
insert_statement = (
insert(PriceModel.__table__)
.values([price.dict() for price in prices])
.on_conflict_do_nothing()
)
db_session.execute(insert_statement)
db_session.commit()

View File

@ -31,6 +31,7 @@ from mev_inspect.crud.traces import (
from mev_inspect.crud.transfers import delete_transfers_for_block, write_transfers
from mev_inspect.liquidations import get_liquidations
from mev_inspect.miner_payments import get_miner_payments
from mev_inspect.punks import get_punk_bid_acceptances, get_punk_bids, get_punk_snipes
from mev_inspect.swaps import get_swaps
from mev_inspect.transfers import get_transfers
@ -90,6 +91,12 @@ async def inspect_block(
await delete_liquidations_for_block(inspect_db_session, block_number)
await write_liquidations(inspect_db_session, liquidations)
punk_bids = get_punk_bids(classified_traces)
punk_bid_acceptances = get_punk_bid_acceptances(classified_traces)
punk_snipes = get_punk_snipes(punk_bids, punk_bid_acceptances)
logger.info(f"Block: {block_number} -- Found {len(punk_snipes)} punk snipes")
miner_payments = get_miner_payments(
block.miner, block.base_fee_per_gas, classified_traces, block.receipts
)

View File

@ -0,0 +1,11 @@
from sqlalchemy import Column, Numeric, String, TIMESTAMP
from .base import Base
class PriceModel(Base):
__tablename__ = "prices"
timestamp = Column(TIMESTAMP, nullable=False, primary_key=True)
usd_price = Column(Numeric, nullable=False)
token_address = Column(String, nullable=False, primary_key=True)

29
mev_inspect/prices.py Normal file
View File

@ -0,0 +1,29 @@
from typing import List
from mev_inspect.classifiers.specs.weth import WETH_ADDRESS
from mev_inspect.coinbase import fetch_coinbase_prices
from mev_inspect.schemas.prices import Price
from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS
SUPPORTED_TOKENS = [
WETH_ADDRESS,
ETH_TOKEN_ADDRESS,
]
async def fetch_all_supported_prices() -> List[Price]:
prices = []
for token_address in SUPPORTED_TOKENS:
coinbase_prices = await fetch_coinbase_prices(token_address)
for usd_price, timestamp_seconds in coinbase_prices.all.prices:
price = Price(
token_address=token_address,
usd_price=usd_price,
timestamp=timestamp_seconds,
)
prices.append(price)
return prices

125
mev_inspect/punks.py Normal file
View File

@ -0,0 +1,125 @@
from typing import List, Optional
from mev_inspect.schemas.traces import (
ClassifiedTrace,
Classification,
DecodedCallTrace,
)
from mev_inspect.schemas.punk_bid import PunkBid
from mev_inspect.schemas.punk_accept_bid import PunkBidAcceptance
from mev_inspect.schemas.punk_snipe import PunkSnipe
from mev_inspect.traces import get_traces_by_transaction_hash
def _get_highest_punk_bid_per_index(
punk_bids: List[PunkBid], punk_index: int
) -> Optional[PunkBid]:
highest_punk_bid = None
for punk_bid in punk_bids:
if punk_bid.punk_index == punk_index:
if highest_punk_bid is None:
highest_punk_bid = punk_bid
elif punk_bid.price > highest_punk_bid.price:
highest_punk_bid = punk_bid
return highest_punk_bid
def get_punk_snipes(
punk_bids: List[PunkBid], punk_bid_acceptances: List[PunkBidAcceptance]
) -> List[PunkSnipe]:
punk_snipe_list = []
for punk_bid_acceptance in punk_bid_acceptances:
highest_punk_bid = _get_highest_punk_bid_per_index(
punk_bids, punk_bid_acceptance.punk_index
)
if highest_punk_bid is None:
continue
if highest_punk_bid.price > punk_bid_acceptance.min_price:
punk_snipe = PunkSnipe(
block_number=highest_punk_bid.block_number,
transaction_hash=highest_punk_bid.transaction_hash,
trace_address=highest_punk_bid.trace_address,
from_address=highest_punk_bid.from_address,
punk_index=highest_punk_bid.punk_index,
min_acceptance_price=punk_bid_acceptance.min_price,
acceptance_price=highest_punk_bid.price,
)
punk_snipe_list.append(punk_snipe)
return punk_snipe_list
def get_punk_bid_acceptances(traces: List[ClassifiedTrace]) -> List[PunkBidAcceptance]:
punk_bid_acceptances = []
for _, transaction_traces in get_traces_by_transaction_hash(traces).items():
punk_bid_acceptances += _get_punk_bid_acceptances_for_transaction(
list(transaction_traces)
)
return punk_bid_acceptances
def _get_punk_bid_acceptances_for_transaction(
traces: List[ClassifiedTrace],
) -> List[PunkBidAcceptance]:
ordered_traces = list(sorted(traces, key=lambda t: t.trace_address))
punk_bid_acceptances = []
for trace in ordered_traces:
if not isinstance(trace, DecodedCallTrace):
continue
elif trace.classification == Classification.punk_accept_bid:
punk_accept_bid = PunkBidAcceptance(
block_number=trace.block_number,
transaction_hash=trace.transaction_hash,
trace_address=trace.trace_address,
from_address=trace.from_address,
punk_index=trace.inputs["punkIndex"],
min_price=trace.inputs["minPrice"],
)
punk_bid_acceptances.append(punk_accept_bid)
return punk_bid_acceptances
def get_punk_bids(traces: List[ClassifiedTrace]) -> List[PunkBid]:
punk_bids = []
for _, transaction_traces in get_traces_by_transaction_hash(traces).items():
punk_bids += _get_punk_bids_for_transaction(list(transaction_traces))
return punk_bids
def _get_punk_bids_for_transaction(traces: List[ClassifiedTrace]) -> List[PunkBid]:
ordered_traces = list(sorted(traces, key=lambda t: t.trace_address))
punk_bids = []
for trace in ordered_traces:
if not isinstance(trace, DecodedCallTrace):
continue
elif trace.classification == Classification.punk_bid:
punk_bid = PunkBid(
transaction_hash=trace.transaction_hash,
block_number=trace.block_number,
trace_address=trace.trace_address,
from_address=trace.from_address,
punk_index=trace.inputs["punkIndex"],
price=trace.value,
)
punk_bids.append(punk_bid)
return punk_bids

View File

@ -0,0 +1,20 @@
from typing import List, Tuple
from pydantic import BaseModel
class CoinbasePricesEntry(BaseModel):
# tuple of price and timestamp
prices: List[Tuple[float, int]]
class CoinbasePrices(BaseModel):
all: CoinbasePricesEntry
class CoinbasePricesDataResponse(BaseModel):
prices: CoinbasePrices
class CoinbasePricesResponse(BaseModel):
data: CoinbasePricesDataResponse

View File

@ -0,0 +1,9 @@
from datetime import datetime
from pydantic import BaseModel
class Price(BaseModel):
token_address: str
timestamp: datetime
usd_price: float

View File

@ -0,0 +1,12 @@
from typing import List
from pydantic import BaseModel
class PunkBidAcceptance(BaseModel):
block_number: int
transaction_hash: str
trace_address: List[int]
from_address: str
punk_index: int
min_price: int

View File

@ -0,0 +1,12 @@
from typing import List
from pydantic import BaseModel
class PunkBid(BaseModel):
block_number: int
transaction_hash: str
trace_address: List[int]
from_address: str
punk_index: int
price: int

View File

@ -0,0 +1,13 @@
from typing import List
from pydantic import BaseModel
class PunkSnipe(BaseModel):
block_number: int
transaction_hash: str
trace_address: List[int]
from_address: str
punk_index: int
min_acceptance_price: int
acceptance_price: int

View File

@ -31,6 +31,8 @@ class Classification(Enum):
transfer = "transfer"
liquidate = "liquidate"
seize = "seize"
punk_bid = "punk_bid"
punk_accept_bid = "punk_accept_bid"
class Protocol(Enum):
@ -44,6 +46,7 @@ class Protocol(Enum):
balancer_v1 = "balancer_v1"
compound_v2 = "compound_v2"
cream = "cream"
cryptopunks = "cryptopunks"
class ClassifiedTrace(Trace):

View File

@ -1,8 +1,8 @@
import json
from hexbytes import HexBytes
from web3.datastructures import AttributeDict
from pydantic import BaseModel
from web3.datastructures import AttributeDict
def to_camel(string: str) -> str:

View File

@ -36,6 +36,7 @@ build-backend = "poetry.core.masonry.api"
inspect-block = 'cli:inspect_block_command'
inspect-many-blocks = 'cli:inspect_many_blocks_command'
fetch-block = 'cli:fetch_block_command'
fetch-all-prices = 'cli:fetch_all_prices'
[tool.black]
exclude = '''

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

129
tests/test_0x.py Normal file
View File

@ -0,0 +1,129 @@
from mev_inspect.schemas.swaps import Swap
from mev_inspect.swaps import get_swaps
from mev_inspect.schemas.traces import Protocol
from mev_inspect.classifiers.trace import TraceClassifier
from tests.utils import load_test_block
def test_fillLimitOrder_swap():
transaction_hash = (
"0xa043976d736ec8dc930c0556dffd0a86a4bfc80bf98fb7995c791fb4dc488b5d"
)
block_number = 13666312
swap = Swap(
abi_name="INativeOrdersFeature",
transaction_hash=transaction_hash,
block_number=block_number,
trace_address=[0, 2, 0, 1],
contract_address="0xdef1c0ded9bec7f1a1670819833240f027b25eff",
from_address="0x00000000000e1d0dabf7b7c7b68866fc940d0db8",
to_address="0xdef1c0ded9bec7f1a1670819833240f027b25eff",
token_in_address="0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2",
token_in_amount=35000000000000000000,
token_out_address="0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48",
token_out_amount=143949683150,
protocol=Protocol.zero_ex,
error=None,
)
block = load_test_block(block_number)
trace_classifier = TraceClassifier()
classified_traces = trace_classifier.classify(block.traces)
result = get_swaps(classified_traces)
assert result.count(swap) == 1
def test__fillLimitOrder_swap():
transaction_hash = (
"0x9255addffa2dbeb9560c5e20e78a78c949488d2054c70b2155c39f9e28394cbf"
)
block_number = 13666184
swap = Swap(
abi_name="INativeOrdersFeature",
transaction_hash=transaction_hash,
block_number=block_number,
trace_address=[0, 1],
contract_address="0xdef1c0ded9bec7f1a1670819833240f027b25eff",
from_address="0xdef1c0ded9bec7f1a1670819833240f027b25eff",
to_address="0xdef1c0ded9bec7f1a1670819833240f027b25eff",
token_in_address="0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48",
token_in_amount=30000000,
token_out_address="0x9ff79c75ae2bcbe0ec63c0375a3ec90ff75bbe0f",
token_out_amount=100000001,
protocol=Protocol.zero_ex,
error=None,
)
block = load_test_block(block_number)
trace_classifier = TraceClassifier()
classified_traces = trace_classifier.classify(block.traces)
result = get_swaps(classified_traces)
assert result.count(swap) == 1
def test_RfqLimitOrder_swap():
transaction_hash = (
"0x1c948eb7c59ddbe6b916cf68f5df86eb44a7c9e728221fcd8ab750f137fd2a0f"
)
block_number = 13666326
swap = Swap(
abi_name="INativeOrdersFeature",
transaction_hash=transaction_hash,
block_number=block_number,
trace_address=[0, 1, 13, 0, 1],
contract_address="0xdef1c0ded9bec7f1a1670819833240f027b25eff",
from_address="0xdef171fe48cf0115b1d80b88dc8eab59176fee57",
to_address="0xdef1c0ded9bec7f1a1670819833240f027b25eff",
token_in_address="0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48",
token_in_amount=288948250430,
token_out_address="0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2",
token_out_amount=70500000000000000000,
protocol=Protocol.zero_ex,
error=None,
)
block = load_test_block(block_number)
trace_classifier = TraceClassifier()
classified_traces = trace_classifier.classify(block.traces)
result = get_swaps(classified_traces)
assert result.count(swap) == 1
def test__RfqLimitOrder_swap():
transaction_hash = (
"0x4f66832e654f8a4d773d9769571155df3722401343247376d6bb56626db29b90"
)
block_number = 13666363
swap = Swap(
abi_name="INativeOrdersFeature",
transaction_hash=transaction_hash,
block_number=block_number,
trace_address=[1, 0, 1, 0, 1],
contract_address="0xdef1c0ded9bec7f1a1670819833240f027b25eff",
from_address="0xdef1c0ded9bec7f1a1670819833240f027b25eff",
to_address="0xdef1c0ded9bec7f1a1670819833240f027b25eff",
token_in_address="0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2",
token_in_amount=979486121594935552,
token_out_address="0x95ad61b0a150d79219dcf64e1e6cc01f0b64c4ce",
token_out_amount=92404351093861841165644172,
protocol=Protocol.zero_ex,
error=None,
)
block = load_test_block(block_number)
trace_classifier = TraceClassifier()
classified_traces = trace_classifier.classify(block.traces)
result = get_swaps(classified_traces)
assert result.count(swap) == 1