diff --git a/cli.py b/cli.py index c112aa0..e1e80c4 100644 --- a/cli.py +++ b/cli.py @@ -5,6 +5,7 @@ import sys import click from web3 import Web3 +from mev_inspect.classifiers.trace import TraceClassifier from mev_inspect.db import get_session from mev_inspect.inspect_block import inspect_block from mev_inspect.provider import get_base_provider @@ -29,11 +30,20 @@ def inspect_block_command(block_number: int, rpc: str, cache: bool): db_session = get_session() base_provider = get_base_provider(rpc) w3 = Web3(base_provider) + trace_classifier = TraceClassifier() if not cache: logger.info("Skipping cache") - inspect_block(db_session, base_provider, w3, block_number, should_cache=cache) + inspect_block( + db_session, + base_provider, + w3, + trace_classifier, + block_number, + should_cache=cache, + ) + @cli.command() @@ -48,6 +58,7 @@ def inspect_many_blocks_command( db_session = get_session() base_provider = get_base_provider(rpc) w3 = Web3(base_provider) + trace_classifier = TraceClassifier() if not cache: logger.info("Skipping cache") @@ -65,6 +76,7 @@ def inspect_many_blocks_command( db_session, base_provider, w3, + trace_classifier, block_number, should_write_classified_traces=False, should_cache=cache, diff --git a/mev_inspect/decode.py b/mev_inspect/decode.py index 19121a5..c06128b 100644 --- a/mev_inspect/decode.py +++ b/mev_inspect/decode.py @@ -2,14 +2,18 @@ from typing import Dict, Optional import eth_utils.abi -from hexbytes import HexBytes from eth_abi import decode_abi from eth_abi.exceptions import InsufficientDataBytes, NonEmptyPaddingBytes +from hexbytes._utils import hexstr_to_bytes from mev_inspect.schemas.abi import ABI, ABIFunctionDescription from mev_inspect.schemas.call_data import CallData +# 0x + 8 characters +SELECTOR_LENGTH = 10 + + class ABIDecoder: def __init__(self, abi: ABI): self._functions_by_selector: Dict[str, ABIFunctionDescription] = { @@ -19,8 +23,7 @@ class ABIDecoder: } def decode(self, data: str) -> Optional[CallData]: - hex_data = HexBytes(data) - selector, params = hex_data[:4], hex_data[4:] + selector, params = data[:SELECTOR_LENGTH], data[SELECTOR_LENGTH:] func = self._functions_by_selector.get(selector) @@ -36,7 +39,7 @@ class ABIDecoder: ] try: - decoded = decode_abi(types, params) + decoded = decode_abi(types, hexstr_to_bytes(params)) except (InsufficientDataBytes, NonEmptyPaddingBytes): return None diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py index 3128fd0..f19cb51 100644 --- a/mev_inspect/inspect_block.py +++ b/mev_inspect/inspect_block.py @@ -36,16 +36,14 @@ def inspect_block( db_session, base_provider, w3: Web3, + trace_clasifier: TraceClassifier, block_number: int, should_cache: bool, should_write_classified_traces: bool = True, - should_write_swaps: bool = True, - should_write_transfers: bool = True, - should_write_arbitrages: bool = True, - should_write_liquidations: bool = True, - should_write_miner_payments: bool = True, ): - block = create_from_block_number(base_provider, w3, block_number, should_cache) + block = create_from_block_number( + base_provider, w3, block_number, should_cache, + ) logger.info(f"Total traces: {len(block.traces)}") @@ -54,7 +52,6 @@ def inspect_block( ) logger.info(f"Total transactions: {total_transactions}") - trace_clasifier = TraceClassifier() classified_traces = trace_clasifier.classify(block.traces) logger.info(f"Returned {len(classified_traces)} classified traces") @@ -63,35 +60,32 @@ def inspect_block( write_classified_traces(db_session, classified_traces) transfers = get_transfers(classified_traces) - if should_write_transfers: - delete_transfers_for_block(db_session, block_number) - write_transfers(db_session, transfers) + logger.info(f"Found {len(transfers)} transfers") + + delete_transfers_for_block(db_session, block_number) + write_transfers(db_session, transfers) swaps = get_swaps(classified_traces) logger.info(f"Found {len(swaps)} swaps") - if should_write_swaps: - delete_swaps_for_block(db_session, block_number) - write_swaps(db_session, swaps) + delete_swaps_for_block(db_session, block_number) + write_swaps(db_session, swaps) arbitrages = get_arbitrages(swaps) logger.info(f"Found {len(arbitrages)} arbitrages") - if should_write_arbitrages: - delete_arbitrages_for_block(db_session, block_number) - write_arbitrages(db_session, arbitrages) + delete_arbitrages_for_block(db_session, block_number) + write_arbitrages(db_session, arbitrages) - liquidations = get_liquidations(classified_traces, w3) + liquidations = get_liquidations(classified_traces) logger.info(f"Found {len(liquidations)} liquidations") - if should_write_liquidations: - delete_liquidations_for_block(db_session, block_number) - write_liquidations(db_session, liquidations) + delete_liquidations_for_block(db_session, block_number) + write_liquidations(db_session, liquidations) miner_payments = get_miner_payments( block.miner, block.base_fee_per_gas, classified_traces, block.receipts ) - if should_write_miner_payments: - delete_miner_payments_for_block(db_session, block_number) - write_miner_payments(db_session, miner_payments) + delete_miner_payments_for_block(db_session, block_number) + write_miner_payments(db_session, miner_payments) diff --git a/mev_inspect/liquidations.py b/mev_inspect/liquidations.py index f0bad7f..e2e0f6e 100644 --- a/mev_inspect/liquidations.py +++ b/mev_inspect/liquidations.py @@ -1,15 +1,9 @@ from typing import List -from web3 import Web3 from mev_inspect.aave_liquidations import get_aave_liquidations -from mev_inspect.compound_liquidations import ( - get_compound_liquidations, - fetch_all_underlying_markets, -) from mev_inspect.schemas.classified_traces import ( ClassifiedTrace, Classification, - Protocol, ) from mev_inspect.schemas.liquidations import Liquidation @@ -23,18 +17,7 @@ def has_liquidations(classified_traces: List[ClassifiedTrace]) -> bool: def get_liquidations( - classified_traces: List[ClassifiedTrace], w3: Web3 + classified_traces: List[ClassifiedTrace], ) -> List[Liquidation]: - # to avoid contract calls to fetch comp/cream markets - # unless there is a liquidation - - if has_liquidations(classified_traces): - aave_liquidations = get_aave_liquidations(classified_traces) - comp_markets = fetch_all_underlying_markets(w3, Protocol.compound_v2) - cream_markets = fetch_all_underlying_markets(w3, Protocol.cream) - compound_liquidations = get_compound_liquidations( - classified_traces, comp_markets, cream_markets - ) - return aave_liquidations + compound_liquidations - - return [] + aave_liquidations = get_aave_liquidations(classified_traces) + return aave_liquidations diff --git a/mev_inspect/schemas/abi.py b/mev_inspect/schemas/abi.py index 9cb16e4..a17b915 100644 --- a/mev_inspect/schemas/abi.py +++ b/mev_inspect/schemas/abi.py @@ -3,7 +3,6 @@ from typing import List, Optional, Union from typing_extensions import Literal import eth_utils.abi -from hexbytes import HexBytes from pydantic import BaseModel from web3 import Web3 @@ -42,9 +41,9 @@ class ABIFunctionDescription(BaseModel): name: str inputs: List[ABIDescriptionInput] - def get_selector(self) -> HexBytes: + def get_selector(self) -> str: signature = self.get_signature() - return Web3.sha3(text=signature)[0:4] + return Web3.sha3(text=signature)[0:4].hex() def get_signature(self) -> str: joined_input_types = ",".join( diff --git a/mev_inspect/utils.py b/mev_inspect/utils.py index 10d331a..922fada 100644 --- a/mev_inspect/utils.py +++ b/mev_inspect/utils.py @@ -1,5 +1,5 @@ -from hexbytes.main import HexBytes +from hexbytes._utils import hexstr_to_bytes def hex_to_int(value: str) -> int: - return int.from_bytes(HexBytes(value), byteorder="big") + return int.from_bytes(hexstr_to_bytes(value), byteorder="big") diff --git a/poetry.lock b/poetry.lock index 0f03c5c..e5df627 100644 --- a/poetry.lock +++ b/poetry.lock @@ -134,6 +134,14 @@ d = ["aiohttp (>=3.6.0)", "aiohttp-cors (>=0.4.0)"] python2 = ["typed-ast (>=1.4.2)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "bottle" +version = "0.12.19" +description = "Fast and simple WSGI-framework for small web-applications." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "certifi" version = "2021.5.30" @@ -199,6 +207,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] toml = ["toml"] +[[package]] +name = "cprofilev" +version = "1.0.7" +description = "An easier way to use cProfile" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +bottle = "*" + [[package]] name = "cytoolz" version = "0.11.0" @@ -1044,7 +1063,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "206acce73eccf4be7eec1ed7b1a0703438601143a107c4285f67730934eed86a" +content-hash = "be169aa0ca2984317d13a45a903eb4a4f7709f4d6b98d9b03f9f779c63b9dd01" [metadata.files] aiohttp = [ @@ -1125,6 +1144,10 @@ black = [ {file = "black-21.7b0-py3-none-any.whl", hash = "sha256:1c7aa6ada8ee864db745b22790a32f94b2795c253a75d6d9b5e439ff10d23116"}, {file = "black-21.7b0.tar.gz", hash = "sha256:c8373c6491de9362e39271630b65b964607bc5c79c83783547d76c839b3aa219"}, ] +bottle = [ + {file = "bottle-0.12.19-py3-none-any.whl", hash = "sha256:f6b8a34fe9aa406f9813c02990db72ca69ce6a158b5b156d2c41f345016a723d"}, + {file = "bottle-0.12.19.tar.gz", hash = "sha256:a9d73ffcbc6a1345ca2d7949638db46349f5b2b77dac65d6494d45c23628da2c"}, +] certifi = [ {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, @@ -1203,6 +1226,9 @@ coverage = [ {file = "coverage-5.5-pp37-none-any.whl", hash = "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4"}, {file = "coverage-5.5.tar.gz", hash = "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c"}, ] +cprofilev = [ + {file = "CProfileV-1.0.7.tar.gz", hash = "sha256:8791748b1f3d3468c2c927c3fd5f905080b84d8f2d217ca764b7d9d7a1fb9a77"}, +] cytoolz = [ {file = "cytoolz-0.11.0-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:c50051c02b23823209d6b0e8f7b2b37371312da50ca78165871dc6fed7bd37df"}, {file = "cytoolz-0.11.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:140eaadcd86216d4a185db3a37396ee80dd2edc6e490ba37a3d7c1b17a124078"}, diff --git a/pyproject.toml b/pyproject.toml index 26607a2..e28aab6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ pytest-cov = "^2.12.1" coverage = "^5.5" alembic = "^1.6.5" black = "^21.7b0" +CProfileV = "^1.0.7" [build-system] requires = ["poetry-core>=1.0.0"]