diff --git a/.gitignore b/.gitignore index fcc9a74..6cc5bdc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ # venv and test cache files env/ -__pycache__ \ No newline at end of file +__pycache__ + +*.swp diff --git a/block.py b/block.py index 1694538..7add9c9 100644 --- a/block.py +++ b/block.py @@ -4,100 +4,103 @@ from typing import List from web3 import Web3 -from schemas import Block +from schemas import Block, BlockCall, BlockCallType cache_directory = './cache' -def get_transaction_hashes(calls: List[dict]) -> List[str]: - result = [] - - for call in calls: - if call['type'] != 'reward': - if call['transactionHash'] in result: - continue - else: - result.append(call['transactionHash']) - - return result - - -def write_json(block: Block): - cache_path = _get_cache_path(block.block_number) - write_mode = "w" if cache_path.is_file() else "x" - - with open(cache_path, mode=write_mode) as cache_file: - cache_file.write(block.json(sort_keys=True, indent=4)) - - ## Creates a block object, either from the cache or from the chain itself ## Note that you need to pass in the provider, not the web3 wrapped provider object! ## This is because only the provider allows you to make json rpc requests def createFromBlockNumber(block_number: int, base_provider) -> Block: cache_path = _get_cache_path(block_number) - - ## Check to see if the data already exists in the cache - ## if it exists load the data from cache - ## If not then get the data from the chain and save it to the cache + if (cache_path.is_file()): print( f'Cache for block {block_number} exists, ' \ 'loading data from cache' ) - block = Block.parse_file(cache_path) - return block + return Block.parse_file(cache_path) else: - w3 = Web3(base_provider) - print(("Cache for block {block_number} did not exist, getting data").format(block_number=block_number)) - - ## Get block data - block_data = w3.eth.get_block(block_number, True) - - ## Get the block receipts - ## TODO: evaluate whether or not this is sufficient or if gas used needs to be converted to a proper big number. - ## In inspect-ts it needed to be converted - block_receipts_raw = base_provider.make_request("eth_getBlockReceipts", [block_number]) - - ## Trace the whole block, return those calls - block_calls = w3.parity.trace_block(block_number) - - ## Get the logs - block_hash = (block_data.hash).hex() - block_logs = w3.eth.get_logs({'blockHash': block_hash}) - - ## Get gas used by individual txs and store them too - txs_gas_data = {} - for transaction in block_data['transactions']: - tx_hash = (transaction.hash).hex() - tx_data = w3.eth.get_transaction(tx_hash) - tx_receipt = w3.eth.wait_for_transaction_receipt(tx_hash) - txs_gas_data[tx_hash] = { - 'gasUsed': tx_receipt['gasUsed'], # fix: why does this return 0 for certain txs? - 'gasPrice': tx_data['gasPrice'], - 'netFeePaid': tx_data['gasPrice'] * tx_receipt['gasUsed'] - } - - transaction_hashes = get_transaction_hashes(block_calls) - - ## Create a new object - block = Block( - block_number=block_number, - data=block_data, - receipts=block_receipts_raw, - calls=block_calls, - logs=block_logs, - transaction_hashes=transaction_hashes, - txs_gas_data=txs_gas_data, + print( + f"Cache for block {block_number} did not exist, getting data" ) - - ## Write the result to a JSON file for loading in the future - write_json(block) + + w3 = Web3(base_provider) + block = fetch_block(w3, base_provider, block_number) + + cache_block(cache_path, block) return block +def fetch_block(w3, base_provider, block_number: int) -> Block: + ## Get block data + block_data = w3.eth.get_block(block_number, True) + + ## Get the block receipts + ## TODO: evaluate whether or not this is sufficient or if gas used needs to be converted to a proper big number. + ## In inspect-ts it needed to be converted + block_receipts_raw = base_provider.make_request("eth_getBlockReceipts", [block_number]) + + ## Trace the whole block, return those calls + block_calls_json = w3.parity.trace_block(block_number) + block_calls = [ + BlockCall(**call_json) + for call_json in block_calls_json + ] + + ## Get the logs + block_hash = (block_data.hash).hex() + block_logs = w3.eth.get_logs({'blockHash': block_hash}) + + ## Get gas used by individual txs and store them too + txs_gas_data = {} + + for transaction in block_data['transactions']: + tx_hash = (transaction.hash).hex() + tx_data = w3.eth.get_transaction(tx_hash) + tx_receipt = w3.eth.wait_for_transaction_receipt(tx_hash) + txs_gas_data[tx_hash] = { + 'gasUsed': tx_receipt['gasUsed'], # fix: why does this return 0 for certain txs? + 'gasPrice': tx_data['gasPrice'], + 'netFeePaid': tx_data['gasPrice'] * tx_receipt['gasUsed'] + } + + transaction_hashes = get_transaction_hashes(block_calls) + + ## Create a new object + return Block( + block_number=block_number, + data=block_data, + receipts=block_receipts_raw, + calls=block_calls, + logs=block_logs, + transaction_hashes=transaction_hashes, + txs_gas_data=txs_gas_data, + ) + + +def get_transaction_hashes(calls: List[BlockCall]) -> List[str]: + result = [] + + for call in calls: + if call.type != BlockCallType.reward: + if call.transaction_hash not in result: + result.append(call.transaction_hash) + + return result + + +def cache_block(cache_path: Path, block: Block): + write_mode = "w" if cache_path.is_file() else "x" + + with open(cache_path, mode=write_mode) as cache_file: + cache_file.write(block.json()) + + def _get_cache_path(block_number: int) -> Path: cache_directory_path = Path(cache_directory) return cache_directory_path / f"{block_number}-new.json" diff --git a/processor.py b/processor.py index 87be050..548d7b5 100644 --- a/processor.py +++ b/processor.py @@ -1,3 +1,5 @@ +from schemas.utils import to_original_json_dict + class Processor: def __init__(self, base_provider, inspectors) -> None: @@ -7,7 +9,10 @@ class Processor: def get_transaction_evaluations(self, block_data): for transaction_hash in block_data.transaction_hashes: calls = block_data.get_filtered_calls(transaction_hash) + calls_json = [ + to_original_json_dict(call) + for call in calls + ] for inspector in self.inspectors: - inspector.inspect(calls) - # print(calls) \ No newline at end of file + inspector.inspect(calls_json) diff --git a/schemas/__init__.py b/schemas/__init__.py index 7e8c285..38980cb 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -1 +1 @@ -from .blocks import Block +from .blocks import Block, BlockCall, BlockCallType diff --git a/schemas/blocks.py b/schemas/blocks.py index 35aa340..18e41b1 100644 --- a/schemas/blocks.py +++ b/schemas/blocks.py @@ -1,28 +1,44 @@ import json +from enum import Enum from typing import Dict, List, Optional -from hexbytes import HexBytes from pydantic import BaseModel -from web3.datastructures import AttributeDict + +from .utils import CamelModel, Web3Model -class Block(BaseModel): +class BlockCallType(Enum): + call = "call" + create = "create" + delegate_call = "delegateCall" + reward = "reward" + suicide = "suicide" + + +class BlockCall(CamelModel): + action: dict + block_hash: str block_number: int - calls: List[dict] + result: Optional[dict] + subtraces: int + trace_address: List[int] + transaction_hash: Optional[str] + transaction_position: Optional[int] + type: BlockCallType + error: Optional[str] + + +class Block(Web3Model): + block_number: int + calls: List[BlockCall] data: dict logs: List[dict] receipts: dict transaction_hashes: List[str] txs_gas_data: Dict[str, dict] - class Config: - json_encoders = { - AttributeDict: dict, - HexBytes: lambda h: h.hex(), - } - - def get_filtered_calls(self, hash: str) -> List[dict]: + def get_filtered_calls(self, hash: str) -> List[BlockCall]: return [ call for call in self.calls - if call["transactionHash"] == hash + if call.transaction_hash == hash ] diff --git a/schemas/utils.py b/schemas/utils.py new file mode 100644 index 0000000..a3cb04b --- /dev/null +++ b/schemas/utils.py @@ -0,0 +1,34 @@ +import json + +from hexbytes import HexBytes +from pydantic import BaseModel +from web3.datastructures import AttributeDict + + +def to_camel(string: str) -> str: + return ''.join( + word.capitalize() if i > 0 else word + for i, word in enumerate(string.split('_')) + ) + + +def to_original_json_dict(model: BaseModel) -> dict: + return json.loads(model.json(by_alias=True, exclude_unset=True)) + + +class Web3Model(BaseModel): + """BaseModel that handles web3's unserializable objects""" + + class Config: + json_encoders = { + AttributeDict: dict, + HexBytes: lambda h: h.hex(), + } + + +class CamelModel(BaseModel): + """BaseModel that translates from camelCase to snake_case""" + + class Config(Web3Model.Config): + alias_generator = to_camel + allow_population_by_field_name = True