From 2c709b9e0d9e4d4f39c447bb6d6c33663fe60349 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Wed, 21 Jul 2021 00:54:27 -0400 Subject: [PATCH 1/6] Add support for decoding ABIs --- mev_inspect/decode.py | 36 ++++++++++++++++++++++++++++++++ mev_inspect/schemas/abi.py | 1 + mev_inspect/schemas/call_data.py | 9 ++++++++ 3 files changed, 46 insertions(+) create mode 100644 mev_inspect/decode.py create mode 100644 mev_inspect/schemas/call_data.py diff --git a/mev_inspect/decode.py b/mev_inspect/decode.py new file mode 100644 index 0000000..fc0a1ec --- /dev/null +++ b/mev_inspect/decode.py @@ -0,0 +1,36 @@ +from typing import Dict, Optional + +from hexbytes import HexBytes +from eth_abi import decode_abi + +from mev_inspect.schemas.abi import ABI, ABIFunctionDescription +from mev_inspect.schemas.call_data import CallData + + +class ABIDecoder: + def __init__(self, abi: ABI): + self._functions_by_selector: Dict[str, ABIFunctionDescription] = { + description.get_selector(): description + for description in abi + if isinstance(description, ABIFunctionDescription) + } + + def decode(self, data: str) -> Optional[CallData]: + hex_data = HexBytes(data) + selector, params = hex_data[:4], hex_data[4:] + + func = self._functions_by_selector.get(selector) + + if func is None: + return None + + names = [input.name for input in func.inputs] + types = [input.type for input in func.inputs] + + decoded = decode_abi(types, params) + + return CallData( + function_name=func.name, + function_signature=func.get_signature(), + inputs={name: value for name, value in zip(names, decoded)}, + ) diff --git a/mev_inspect/schemas/abi.py b/mev_inspect/schemas/abi.py index 3c54837..7d91130 100644 --- a/mev_inspect/schemas/abi.py +++ b/mev_inspect/schemas/abi.py @@ -24,6 +24,7 @@ NON_FUNCTION_DESCRIPTION_TYPES = Union[ class ABIDescriptionInput(BaseModel): + name: str type: str diff --git a/mev_inspect/schemas/call_data.py b/mev_inspect/schemas/call_data.py new file mode 100644 index 0000000..0ddd9e3 --- /dev/null +++ b/mev_inspect/schemas/call_data.py @@ -0,0 +1,9 @@ +from typing import Any, Dict + +from pydantic import BaseModel + + +class CallData(BaseModel): + function_name: str + function_signature: str + inputs: Dict[str, Any] From eef61b372d9357bd8a1b9906c6adf6ddfab3f40f Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Tue, 20 Jul 2021 18:59:47 -0400 Subject: [PATCH 2/6] Add Inspector and Classification base classes --- mev_inspect/inspectors/__init__.py | 0 mev_inspect/inspectors/base.py | 11 +++++++++++ mev_inspect/schemas/classifications.py | 5 +++++ 3 files changed, 16 insertions(+) create mode 100644 mev_inspect/inspectors/__init__.py create mode 100644 mev_inspect/inspectors/base.py create mode 100644 mev_inspect/schemas/classifications.py diff --git a/mev_inspect/inspectors/__init__.py b/mev_inspect/inspectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mev_inspect/inspectors/base.py b/mev_inspect/inspectors/base.py new file mode 100644 index 0000000..2bb315f --- /dev/null +++ b/mev_inspect/inspectors/base.py @@ -0,0 +1,11 @@ +from abc import ABC, abstractmethod +from typing import Optional + +from mev_inspect.schemas.blocks import NestedTrace +from mev_inspect.schemas.classifications import Classification + + +class Inspector(ABC): + @abstractmethod + def inspect(self, nested_trace: NestedTrace) -> Optional[Classification]: + pass diff --git a/mev_inspect/schemas/classifications.py b/mev_inspect/schemas/classifications.py new file mode 100644 index 0000000..8a48e2f --- /dev/null +++ b/mev_inspect/schemas/classifications.py @@ -0,0 +1,5 @@ +from pydantic import BaseModel + + +class Classification(BaseModel): + pass From 31022e3e453274512693d569fcb0d9465d49dc2a Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Tue, 20 Jul 2021 19:17:15 -0400 Subject: [PATCH 3/6] Move uniswap inspector. Implement the inspector interface --- mev_inspect/inspector_uniswap.py | 96 ------------------------------ mev_inspect/inspectors/__init__.py | 1 + 2 files changed, 1 insertion(+), 96 deletions(-) delete mode 100644 mev_inspect/inspector_uniswap.py diff --git a/mev_inspect/inspector_uniswap.py b/mev_inspect/inspector_uniswap.py deleted file mode 100644 index f64933d..0000000 --- a/mev_inspect/inspector_uniswap.py +++ /dev/null @@ -1,96 +0,0 @@ -import json - -from web3 import Web3 - -from mev_inspect import utils -from mev_inspect.config import load_config - -config = load_config() - -uniswap_router_abi = json.loads(config["ABI"]["UniswapV2Router"]) -uniswap_router_address = config["ADDRESSES"]["UniswapV2Router"] -sushiswap_router_address = config["ADDRESSES"]["SushiswapV2Router"] - -uniswap_pair_abi = json.loads(config["ABI"]["UniswapV2Pair"]) - - -class UniswapInspector: - def __init__(self, base_provider) -> None: - self.w3 = Web3(base_provider) - - self.trading_functions = self.get_trading_functions() - self.uniswap_v2_router_contract = self.w3.eth.contract( - abi=uniswap_router_abi, address=uniswap_router_address - ) - self.uniswap_router_trade_signatures = self.get_router_signatures() - - self.uniswap_v2_pair_contract = self.w3.eth.contract(abi=uniswap_pair_abi) - self.uniswap_v2_pair_swap_signatures = ( - self.uniswap_v2_pair_contract.functions.swap( - 0, 0, uniswap_router_address, "" - ).selector - ) ## Note the address here doesn't matter, but it must be filled out - self.uniswap_v2_pair_reserves_signatures = ( - self.uniswap_v2_pair_contract.functions.getReserves().selector - ) ## Called "checksigs" in mev-inpsect.ts - - print("Built Uniswap inspector") - - def get_trading_functions(self): - ## Gets all functions used for swapping - result = [] - - ## For each entry in the ABI - for abi in uniswap_router_abi: - ## Check to see if the entry is a function and if it is if the function's name starts with swap - if abi["type"] == "function" and abi["name"].startswith("swap"): - ## If so add it to our array - result.append(abi["name"]) - - return result - - def get_router_signatures(self): - ## Gets the selector / function signatures of all the router swap functions - result = [] - - ## For each entry in the ABI - for abi in uniswap_router_abi: - ## Check to see if the entry is a function and if it is if the function's name starts with swap - if abi["type"] == "function" and abi["name"].startswith("swap"): - ## Add a parantheses - function = abi["name"] + "(" - - ## For each input in the function's input - for input in abi["inputs"]: - - ## Concat them into a string with commas - function = function + input["internalType"] + "," - - ## Take off the last comma, add a ')' to close the parentheses - function = function[:-1] + ")" - - ## The result looks like this: 'swapETHForExactTokens(uint256,address[],address,uint256)' - - ## Take the first 4 bytes of the sha3 hash of the above string. - selector = Web3.sha3(text=function)[0:4] - - ## Add that to an array - result.append(selector) - - return result - - def inspect(self, calls): - for call in calls: - print("\n", call) - if ( - call["type"] == "call" - and ( - call["action"]["to"] == uniswap_router_address.lower() - or call["action"]["to"] == sushiswap_router_address.lower() - ) - and utils.check_trace_for_signature( - call, self.uniswap_router_trade_signatures - ) - ): - # print("WIP, here is where there is a call that matches what we are looking for") - 1 == 1 diff --git a/mev_inspect/inspectors/__init__.py b/mev_inspect/inspectors/__init__.py index e69de29..4ece86a 100644 --- a/mev_inspect/inspectors/__init__.py +++ b/mev_inspect/inspectors/__init__.py @@ -0,0 +1 @@ +from .base import Inspector From fd1b11927c795214aa767086d4402492c1e2c6c1 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Tue, 20 Jul 2021 19:34:10 -0400 Subject: [PATCH 4/6] Support Inspector interface in processor. Support it in uniswap inspector --- mev_inspect/inspectors/uniswap.py | 103 +++++++++++++++++++++++++ mev_inspect/processor.py | 48 +++++++++--- mev_inspect/schemas/classifications.py | 9 +++ testing_file.py | 4 +- 4 files changed, 152 insertions(+), 12 deletions(-) create mode 100644 mev_inspect/inspectors/uniswap.py diff --git a/mev_inspect/inspectors/uniswap.py b/mev_inspect/inspectors/uniswap.py new file mode 100644 index 0000000..20aacda --- /dev/null +++ b/mev_inspect/inspectors/uniswap.py @@ -0,0 +1,103 @@ +import json +from typing import Optional + +from web3 import Web3 + +from mev_inspect import utils +from mev_inspect.config import load_config +from mev_inspect.schemas.blocks import NestedTrace, TraceType +from mev_inspect.schemas.classifications import Classification + +from .base import Inspector + +config = load_config() + +uniswap_router_abi = json.loads(config["ABI"]["UniswapV2Router"]) +uniswap_router_address = config["ADDRESSES"]["UniswapV2Router"] +sushiswap_router_address = config["ADDRESSES"]["SushiswapV2Router"] + +uniswap_pair_abi = json.loads(config["ABI"]["UniswapV2Pair"]) + + +class UniswapInspector(Inspector): + def __init__(self, base_provider) -> None: + self.w3 = Web3(base_provider) + + self.trading_functions = self.get_trading_functions() + self.uniswap_v2_router_contract = self.w3.eth.contract( + abi=uniswap_router_abi, address=uniswap_router_address + ) + self.uniswap_router_trade_signatures = self.get_router_signatures() + + self.uniswap_v2_pair_contract = self.w3.eth.contract(abi=uniswap_pair_abi) + self.uniswap_v2_pair_swap_signatures = ( + self.uniswap_v2_pair_contract.functions.swap( + 0, 0, uniswap_router_address, "" + ).selector + ) ## Note the address here doesn't matter, but it must be filled out + self.uniswap_v2_pair_reserves_signatures = ( + self.uniswap_v2_pair_contract.functions.getReserves().selector + ) ## Called "checksigs" in mev-inpsect.ts + + print("Built Uniswap inspector") + + def get_trading_functions(self): + ## Gets all functions used for swapping + result = [] + + ## For each entry in the ABI + for abi in uniswap_router_abi: + ## Check to see if the entry is a function and if it is if the function's name starts with swap + if abi["type"] == "function" and abi["name"].startswith("swap"): + ## If so add it to our array + result.append(abi["name"]) + + return result + + def get_router_signatures(self): + ## Gets the selector / function signatures of all the router swap functions + result = [] + + ## For each entry in the ABI + for abi in uniswap_router_abi: + ## Check to see if the entry is a function and if it is if the function's name starts with swap + if abi["type"] == "function" and abi["name"].startswith("swap"): + ## Add a parantheses + function = abi["name"] + "(" + + ## For each input in the function's input + for input in abi["inputs"]: + + ## Concat them into a string with commas + function = function + input["internalType"] + "," + + ## Take off the last comma, add a ')' to close the parentheses + function = function[:-1] + ")" + + ## The result looks like this: 'swapETHForExactTokens(uint256,address[],address,uint256)' + + ## Take the first 4 bytes of the sha3 hash of the above string. + selector = Web3.sha3(text=function)[0:4] + + ## Add that to an array + result.append(selector) + + return result + + def inspect(self, nested_trace: NestedTrace) -> Optional[Classification]: + trace = nested_trace.trace + + if ( + trace.type == TraceType.call + and ( + trace.action["to"] == uniswap_router_address.lower() + or trace.action["to"] == sushiswap_router_address.lower() + ) + and utils.check_trace_for_signature( + trace, self.uniswap_router_trade_signatures + ) + ): + # print("WIP, here is where there is a call that matches what we are looking for") + 1 == 1 + + return None diff --git a/mev_inspect/processor.py b/mev_inspect/processor.py index d3eca33..6efa84d 100644 --- a/mev_inspect/processor.py +++ b/mev_inspect/processor.py @@ -1,15 +1,43 @@ -from mev_inspect.schemas.utils import to_original_json_dict +from typing import List + +from mev_inspect.inspectors import Inspector +from mev_inspect.schemas.blocks import Block, NestedTrace, TraceType +from mev_inspect.schemas.classifications import ( + Classification, + UnknownClassification, +) +from mev_inspect.traces import as_nested_traces class Processor: - def __init__(self, base_provider, inspectors) -> None: - self.base_provider = base_provider - self.inspectors = inspectors + def __init__(self, inspectors: List[Inspector]) -> None: + self._inspectors = inspectors - def get_transaction_evaluations(self, block_data): - for transaction_hash in block_data.transaction_hashes: - traces = block_data.get_filtered_traces(transaction_hash) - traces_json = [to_original_json_dict(trace) for trace in traces] + def get_transaction_evaluations( + self, + block: Block, + ) -> List[Classification]: + transaction_traces = ( + trace for trace in block.traces if trace.type != TraceType.reward + ) - for inspector in self.inspectors: - inspector.inspect(traces_json) + return [ + self._run_inspectors(nested_trace) + for nested_trace in as_nested_traces(transaction_traces) + ] + + def _run_inspectors(self, nested_trace: NestedTrace) -> Classification: + for inspector in self._inspectors: + classification = inspector.inspect(nested_trace) + + if classification is not None: + return classification + + internal_classifications = [ + self._run_inspectors(subtrace) for subtrace in nested_trace.subtraces + ] + + return UnknownClassification( + trace=nested_trace.trace, + internal_classifications=internal_classifications, + ) diff --git a/mev_inspect/schemas/classifications.py b/mev_inspect/schemas/classifications.py index 8a48e2f..3e0875f 100644 --- a/mev_inspect/schemas/classifications.py +++ b/mev_inspect/schemas/classifications.py @@ -1,5 +1,14 @@ +from typing import List + from pydantic import BaseModel +from .blocks import Trace + class Classification(BaseModel): pass + + +class UnknownClassification(Classification): + trace: Trace + internal_classifications: List[Classification] diff --git a/testing_file.py b/testing_file.py index 0b392a8..8e824dd 100644 --- a/testing_file.py +++ b/testing_file.py @@ -3,7 +3,7 @@ import argparse from web3 import Web3 from mev_inspect import block -from mev_inspect.inspector_uniswap import UniswapInspector +from mev_inspect.inspectors.uniswap import UniswapInspector from mev_inspect.processor import Processor parser = argparse.ArgumentParser(description="Inspect some blocks.") @@ -29,6 +29,6 @@ block_data = block.create_from_block_number(args.block_number[0], base_provider) uniswap_inspector = UniswapInspector(base_provider) ## Create a processor, pass in an ARRAY of inspects -processor = Processor(base_provider, [uniswap_inspector, uniswap_inspector]) +processor = Processor([uniswap_inspector, uniswap_inspector]) processor.get_transaction_evaluations(block_data) From 0e45f22de8b9e44b16f08ab832e0da39d711b151 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Tue, 20 Jul 2021 19:40:44 -0400 Subject: [PATCH 5/6] Add output to testing file to make sure all looks good --- testing_file.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/testing_file.py b/testing_file.py index 8e824dd..080284f 100644 --- a/testing_file.py +++ b/testing_file.py @@ -24,6 +24,12 @@ base_provider = Web3.HTTPProvider(args.rpc) ## Get block data that we need block_data = block.create_from_block_number(args.block_number[0], base_provider) +print(f"Total traces: {len(block_data.traces)}") + +total_transactions = len( + set(t.transaction_hash for t in block_data.traces if t.transaction_hash is not None) +) +print(f"Total transactions: {total_transactions}") ## Build a Uniswap inspector uniswap_inspector = UniswapInspector(base_provider) @@ -31,4 +37,5 @@ uniswap_inspector = UniswapInspector(base_provider) ## Create a processor, pass in an ARRAY of inspects processor = Processor([uniswap_inspector, uniswap_inspector]) -processor.get_transaction_evaluations(block_data) +classifications = processor.get_transaction_evaluations(block_data) +print(f"Returned {len(classifications)} classifications") From b99f7b5aa8934f08656ea77073b490d0537ef3da Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Wed, 21 Jul 2021 12:30:57 -0400 Subject: [PATCH 6/6] Trace is an object --- mev_inspect/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mev_inspect/utils.py b/mev_inspect/utils.py index d5008ad..0f7c094 100644 --- a/mev_inspect/utils.py +++ b/mev_inspect/utils.py @@ -2,14 +2,16 @@ from typing import List from hexbytes.main import HexBytes +from mev_inspect.schemas.blocks import Trace -def check_trace_for_signature(trace: dict, signatures: List[str]): - if trace["action"]["input"] == None: + +def check_trace_for_signature(trace: Trace, signatures: List[str]): + if trace.action["input"] == None: return False ## Iterate over all signatures, and if our trace matches any of them set it to True for signature in signatures: - if HexBytes(trace["action"]["input"]).startswith(signature): + if HexBytes(trace.action["input"]).startswith(signature): ## Note that we are turning the input into hex bytes here, which seems to be fine ## Working with strings was doing weird things return True