crowetic/mev-inspect-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add some small optimizations. Skip compound liquidations for now

Browse Source
This commit is contained in:
Luke Van Seters 2021-10-16 19:40:56 -04:00
parent 3934004ed4
commit 8d9f860346
8 changed files with 72 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cli.py
View File

@ -5,6 +5,7 @@ import sys
import click
from web3 import Web3
from mev_inspect.classifiers.trace import TraceClassifier
from mev_inspect.db import get_session
from mev_inspect.inspect_block import inspect_block
from mev_inspect.provider import get_base_provider
@ -29,11 +30,20 @@ def inspect_block_command(block_number: int, rpc: str, cache: bool):
db_session = get_session()
base_provider = get_base_provider(rpc)
w3 = Web3(base_provider)
trace_classifier = TraceClassifier()
if not cache:
logger.info("Skipping cache")
inspect_block(db_session, base_provider, w3, block_number, should_cache=cache)
inspect_block(
db_session,
base_provider,
w3,
trace_classifier,
block_number,
should_cache=cache,
)
@cli.command()
@ -48,6 +58,7 @@ def inspect_many_blocks_command(
db_session = get_session()
base_provider = get_base_provider(rpc)
w3 = Web3(base_provider)
trace_classifier = TraceClassifier()
if not cache:
logger.info("Skipping cache")
@ -65,6 +76,7 @@ def inspect_many_blocks_command(
db_session,
base_provider,
w3,
trace_classifier,
block_number,
should_write_classified_traces=False,
should_cache=cache,

11
mev_inspect/decode.py
View File

@ -2,14 +2,18 @@ from typing import Dict, Optional
import eth_utils.abi
from hexbytes import HexBytes
from eth_abi import decode_abi
from eth_abi.exceptions import InsufficientDataBytes, NonEmptyPaddingBytes
from hexbytes._utils import hexstr_to_bytes
from mev_inspect.schemas.abi import ABI, ABIFunctionDescription
from mev_inspect.schemas.call_data import CallData
# 0x + 8 characters
SELECTOR_LENGTH = 10
class ABIDecoder:
def __init__(self, abi: ABI):
self._functions_by_selector: Dict[str, ABIFunctionDescription] = {
@ -19,8 +23,7 @@ class ABIDecoder:
}
def decode(self, data: str) -> Optional[CallData]:
hex_data = HexBytes(data)
selector, params = hex_data[:4], hex_data[4:]
selector, params = data[:SELECTOR_LENGTH], data[SELECTOR_LENGTH:]
func = self._functions_by_selector.get(selector)
@ -36,7 +39,7 @@ class ABIDecoder:
]
try:
decoded = decode_abi(types, params)
decoded = decode_abi(types, hexstr_to_bytes(params))
except (InsufficientDataBytes, NonEmptyPaddingBytes):
return None

40
mev_inspect/inspect_block.py
View File

@ -36,16 +36,14 @@ def inspect_block(
db_session,
base_provider,
w3: Web3,
trace_clasifier: TraceClassifier,
block_number: int,
should_cache: bool,
should_write_classified_traces: bool = True,
should_write_swaps: bool = True,
should_write_transfers: bool = True,
should_write_arbitrages: bool = True,
should_write_liquidations: bool = True,
should_write_miner_payments: bool = True,
):
block = create_from_block_number(base_provider, w3, block_number, should_cache)
block = create_from_block_number(
base_provider, w3, block_number, should_cache,
)
logger.info(f"Total traces: {len(block.traces)}")
@ -54,7 +52,6 @@ def inspect_block(
)
logger.info(f"Total transactions: {total_transactions}")
trace_clasifier = TraceClassifier()
classified_traces = trace_clasifier.classify(block.traces)
logger.info(f"Returned {len(classified_traces)} classified traces")
@ -63,35 +60,32 @@ def inspect_block(
write_classified_traces(db_session, classified_traces)
transfers = get_transfers(classified_traces)
if should_write_transfers:
delete_transfers_for_block(db_session, block_number)
write_transfers(db_session, transfers)
logger.info(f"Found {len(transfers)} transfers")
delete_transfers_for_block(db_session, block_number)
write_transfers(db_session, transfers)
swaps = get_swaps(classified_traces)
logger.info(f"Found {len(swaps)} swaps")
if should_write_swaps:
delete_swaps_for_block(db_session, block_number)
write_swaps(db_session, swaps)
delete_swaps_for_block(db_session, block_number)
write_swaps(db_session, swaps)
arbitrages = get_arbitrages(swaps)
logger.info(f"Found {len(arbitrages)} arbitrages")
if should_write_arbitrages:
delete_arbitrages_for_block(db_session, block_number)
write_arbitrages(db_session, arbitrages)
delete_arbitrages_for_block(db_session, block_number)
write_arbitrages(db_session, arbitrages)
liquidations = get_liquidations(classified_traces, w3)
liquidations = get_liquidations(classified_traces)
logger.info(f"Found {len(liquidations)} liquidations")
if should_write_liquidations:
delete_liquidations_for_block(db_session, block_number)
write_liquidations(db_session, liquidations)
delete_liquidations_for_block(db_session, block_number)
write_liquidations(db_session, liquidations)
miner_payments = get_miner_payments(
block.miner, block.base_fee_per_gas, classified_traces, block.receipts
)
if should_write_miner_payments:
delete_miner_payments_for_block(db_session, block_number)
write_miner_payments(db_session, miner_payments)
delete_miner_payments_for_block(db_session, block_number)
write_miner_payments(db_session, miner_payments)

23
mev_inspect/liquidations.py
View File

@ -1,15 +1,9 @@
from typing import List
from web3 import Web3
from mev_inspect.aave_liquidations import get_aave_liquidations
from mev_inspect.compound_liquidations import (
get_compound_liquidations,
fetch_all_underlying_markets,
)
from mev_inspect.schemas.classified_traces import (
ClassifiedTrace,
Classification,
Protocol,
)
from mev_inspect.schemas.liquidations import Liquidation
@ -23,18 +17,7 @@ def has_liquidations(classified_traces: List[ClassifiedTrace]) -> bool:
def get_liquidations(
classified_traces: List[ClassifiedTrace], w3: Web3
classified_traces: List[ClassifiedTrace],
) -> List[Liquidation]:
# to avoid contract calls to fetch comp/cream markets
# unless there is a liquidation
if has_liquidations(classified_traces):
aave_liquidations = get_aave_liquidations(classified_traces)
comp_markets = fetch_all_underlying_markets(w3, Protocol.compound_v2)
cream_markets = fetch_all_underlying_markets(w3, Protocol.cream)
compound_liquidations = get_compound_liquidations(
classified_traces, comp_markets, cream_markets
)
return aave_liquidations + compound_liquidations
return []
aave_liquidations = get_aave_liquidations(classified_traces)
return aave_liquidations

5
mev_inspect/schemas/abi.py
View File

@ -3,7 +3,6 @@ from typing import List, Optional, Union
from typing_extensions import Literal
import eth_utils.abi
from hexbytes import HexBytes
from pydantic import BaseModel
from web3 import Web3
@ -42,9 +41,9 @@ class ABIFunctionDescription(BaseModel):
name: str
inputs: List[ABIDescriptionInput]
def get_selector(self) -> HexBytes:
def get_selector(self) -> str:
signature = self.get_signature()
return Web3.sha3(text=signature)[0:4]
return Web3.sha3(text=signature)[0:4].hex()
def get_signature(self) -> str:
joined_input_types = ",".join(

4
mev_inspect/utils.py
View File

@ -1,5 +1,5 @@
from hexbytes.main import HexBytes
from hexbytes._utils import hexstr_to_bytes
def hex_to_int(value: str) -> int:
return int.from_bytes(HexBytes(value), byteorder="big")
return int.from_bytes(hexstr_to_bytes(value), byteorder="big")

28
poetry.lock generated
View File

@ -134,6 +134,14 @@ d = ["aiohttp (>=3.6.0)", "aiohttp-cors (>=0.4.0)"]
python2 = ["typed-ast (>=1.4.2)"]
uvloop = ["uvloop (>=0.15.2)"]
[[package]]
name = "bottle"
version = "0.12.19"
description = "Fast and simple WSGI-framework for small web-applications."
category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "certifi"
version = "2021.5.30"
@ -199,6 +207,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
[package.extras]
toml = ["toml"]
[[package]]
name = "cprofilev"
version = "1.0.7"
description = "An easier way to use cProfile"
category = "dev"
optional = false
python-versions = "*"
[package.dependencies]
bottle = "*"
[[package]]
name = "cytoolz"
version = "0.11.0"
@ -1044,7 +1063,7 @@ multidict = ">=4.0"
[metadata]
lock-version = "1.1"
python-versions = "^3.9"
content-hash = "206acce73eccf4be7eec1ed7b1a0703438601143a107c4285f67730934eed86a"
content-hash = "be169aa0ca2984317d13a45a903eb4a4f7709f4d6b98d9b03f9f779c63b9dd01"
[metadata.files]
aiohttp = [
@ -1125,6 +1144,10 @@ black = [
{file = "black-21.7b0-py3-none-any.whl", hash = "sha256:1c7aa6ada8ee864db745b22790a32f94b2795c253a75d6d9b5e439ff10d23116"},
{file = "black-21.7b0.tar.gz", hash = "sha256:c8373c6491de9362e39271630b65b964607bc5c79c83783547d76c839b3aa219"},
]
bottle = [
{file = "bottle-0.12.19-py3-none-any.whl", hash = "sha256:f6b8a34fe9aa406f9813c02990db72ca69ce6a158b5b156d2c41f345016a723d"},
{file = "bottle-0.12.19.tar.gz", hash = "sha256:a9d73ffcbc6a1345ca2d7949638db46349f5b2b77dac65d6494d45c23628da2c"},
]
certifi = [
{file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"},
{file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"},
@ -1203,6 +1226,9 @@ coverage = [
{file = "coverage-5.5-pp37-none-any.whl", hash = "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4"},
{file = "coverage-5.5.tar.gz", hash = "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c"},
]
cprofilev = [
{file = "CProfileV-1.0.7.tar.gz", hash = "sha256:8791748b1f3d3468c2c927c3fd5f905080b84d8f2d217ca764b7d9d7a1fb9a77"},
]
cytoolz = [
{file = "cytoolz-0.11.0-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:c50051c02b23823209d6b0e8f7b2b37371312da50ca78165871dc6fed7bd37df"},
{file = "cytoolz-0.11.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:140eaadcd86216d4a185db3a37396ee80dd2edc6e490ba37a3d7c1b17a124078"},

1
pyproject.toml
View File

@ -23,6 +23,7 @@ pytest-cov = "^2.12.1"
coverage = "^5.5"
alembic = "^1.6.5"
black = "^21.7b0"
CProfileV = "^1.0.7"
[build-system]
requires = ["poetry-core>=1.0.0"]