From 6020e48c31f6f8cb8a3ac4274c8d07b5503d090e Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 12 Jul 2021 15:12:23 -0400 Subject: [PATCH 1/3] Add BlockCall model. Use it in Block --- block.py | 147 ++++++++++++++++++++++---------------------- processor.py | 9 ++- schemas/__init__.py | 2 +- schemas/blocks.py | 30 ++++++++- schemas/utils.py | 22 +++++++ 5 files changed, 132 insertions(+), 78 deletions(-) create mode 100644 schemas/utils.py diff --git a/block.py b/block.py index 1694538..7add9c9 100644 --- a/block.py +++ b/block.py @@ -4,100 +4,103 @@ from typing import List from web3 import Web3 -from schemas import Block +from schemas import Block, BlockCall, BlockCallType cache_directory = './cache' -def get_transaction_hashes(calls: List[dict]) -> List[str]: - result = [] - - for call in calls: - if call['type'] != 'reward': - if call['transactionHash'] in result: - continue - else: - result.append(call['transactionHash']) - - return result - - -def write_json(block: Block): - cache_path = _get_cache_path(block.block_number) - write_mode = "w" if cache_path.is_file() else "x" - - with open(cache_path, mode=write_mode) as cache_file: - cache_file.write(block.json(sort_keys=True, indent=4)) - - ## Creates a block object, either from the cache or from the chain itself ## Note that you need to pass in the provider, not the web3 wrapped provider object! ## This is because only the provider allows you to make json rpc requests def createFromBlockNumber(block_number: int, base_provider) -> Block: cache_path = _get_cache_path(block_number) - - ## Check to see if the data already exists in the cache - ## if it exists load the data from cache - ## If not then get the data from the chain and save it to the cache + if (cache_path.is_file()): print( f'Cache for block {block_number} exists, ' \ 'loading data from cache' ) - block = Block.parse_file(cache_path) - return block + return Block.parse_file(cache_path) else: - w3 = Web3(base_provider) - print(("Cache for block {block_number} did not exist, getting data").format(block_number=block_number)) - - ## Get block data - block_data = w3.eth.get_block(block_number, True) - - ## Get the block receipts - ## TODO: evaluate whether or not this is sufficient or if gas used needs to be converted to a proper big number. - ## In inspect-ts it needed to be converted - block_receipts_raw = base_provider.make_request("eth_getBlockReceipts", [block_number]) - - ## Trace the whole block, return those calls - block_calls = w3.parity.trace_block(block_number) - - ## Get the logs - block_hash = (block_data.hash).hex() - block_logs = w3.eth.get_logs({'blockHash': block_hash}) - - ## Get gas used by individual txs and store them too - txs_gas_data = {} - for transaction in block_data['transactions']: - tx_hash = (transaction.hash).hex() - tx_data = w3.eth.get_transaction(tx_hash) - tx_receipt = w3.eth.wait_for_transaction_receipt(tx_hash) - txs_gas_data[tx_hash] = { - 'gasUsed': tx_receipt['gasUsed'], # fix: why does this return 0 for certain txs? - 'gasPrice': tx_data['gasPrice'], - 'netFeePaid': tx_data['gasPrice'] * tx_receipt['gasUsed'] - } - - transaction_hashes = get_transaction_hashes(block_calls) - - ## Create a new object - block = Block( - block_number=block_number, - data=block_data, - receipts=block_receipts_raw, - calls=block_calls, - logs=block_logs, - transaction_hashes=transaction_hashes, - txs_gas_data=txs_gas_data, + print( + f"Cache for block {block_number} did not exist, getting data" ) - - ## Write the result to a JSON file for loading in the future - write_json(block) + + w3 = Web3(base_provider) + block = fetch_block(w3, base_provider, block_number) + + cache_block(cache_path, block) return block +def fetch_block(w3, base_provider, block_number: int) -> Block: + ## Get block data + block_data = w3.eth.get_block(block_number, True) + + ## Get the block receipts + ## TODO: evaluate whether or not this is sufficient or if gas used needs to be converted to a proper big number. + ## In inspect-ts it needed to be converted + block_receipts_raw = base_provider.make_request("eth_getBlockReceipts", [block_number]) + + ## Trace the whole block, return those calls + block_calls_json = w3.parity.trace_block(block_number) + block_calls = [ + BlockCall(**call_json) + for call_json in block_calls_json + ] + + ## Get the logs + block_hash = (block_data.hash).hex() + block_logs = w3.eth.get_logs({'blockHash': block_hash}) + + ## Get gas used by individual txs and store them too + txs_gas_data = {} + + for transaction in block_data['transactions']: + tx_hash = (transaction.hash).hex() + tx_data = w3.eth.get_transaction(tx_hash) + tx_receipt = w3.eth.wait_for_transaction_receipt(tx_hash) + txs_gas_data[tx_hash] = { + 'gasUsed': tx_receipt['gasUsed'], # fix: why does this return 0 for certain txs? + 'gasPrice': tx_data['gasPrice'], + 'netFeePaid': tx_data['gasPrice'] * tx_receipt['gasUsed'] + } + + transaction_hashes = get_transaction_hashes(block_calls) + + ## Create a new object + return Block( + block_number=block_number, + data=block_data, + receipts=block_receipts_raw, + calls=block_calls, + logs=block_logs, + transaction_hashes=transaction_hashes, + txs_gas_data=txs_gas_data, + ) + + +def get_transaction_hashes(calls: List[BlockCall]) -> List[str]: + result = [] + + for call in calls: + if call.type != BlockCallType.reward: + if call.transaction_hash not in result: + result.append(call.transaction_hash) + + return result + + +def cache_block(cache_path: Path, block: Block): + write_mode = "w" if cache_path.is_file() else "x" + + with open(cache_path, mode=write_mode) as cache_file: + cache_file.write(block.json()) + + def _get_cache_path(block_number: int) -> Path: cache_directory_path = Path(cache_directory) return cache_directory_path / f"{block_number}-new.json" diff --git a/processor.py b/processor.py index 87be050..548d7b5 100644 --- a/processor.py +++ b/processor.py @@ -1,3 +1,5 @@ +from schemas.utils import to_original_json_dict + class Processor: def __init__(self, base_provider, inspectors) -> None: @@ -7,7 +9,10 @@ class Processor: def get_transaction_evaluations(self, block_data): for transaction_hash in block_data.transaction_hashes: calls = block_data.get_filtered_calls(transaction_hash) + calls_json = [ + to_original_json_dict(call) + for call in calls + ] for inspector in self.inspectors: - inspector.inspect(calls) - # print(calls) \ No newline at end of file + inspector.inspect(calls_json) diff --git a/schemas/__init__.py b/schemas/__init__.py index 7e8c285..38980cb 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -1 +1 @@ -from .blocks import Block +from .blocks import Block, BlockCall, BlockCallType diff --git a/schemas/blocks.py b/schemas/blocks.py index 35aa340..fb71d97 100644 --- a/schemas/blocks.py +++ b/schemas/blocks.py @@ -1,14 +1,38 @@ import json +from enum import Enum from typing import Dict, List, Optional from hexbytes import HexBytes from pydantic import BaseModel from web3.datastructures import AttributeDict +from .utils import CamelModel + + +class BlockCallType(Enum): + call = "call" + create = "create" + delegate_call = "delegateCall" + reward = "reward" + suicide = "suicide" + + +class BlockCall(CamelModel): + action: dict + block_hash: str + block_number: int + result: Optional[dict] + subtraces: int + trace_address: List[int] + transaction_hash: Optional[str] + transaction_position: Optional[int] + type: BlockCallType + error: Optional[str] + class Block(BaseModel): block_number: int - calls: List[dict] + calls: List[BlockCall] data: dict logs: List[dict] receipts: dict @@ -21,8 +45,8 @@ class Block(BaseModel): HexBytes: lambda h: h.hex(), } - def get_filtered_calls(self, hash: str) -> List[dict]: + def get_filtered_calls(self, hash: str) -> List[BlockCall]: return [ call for call in self.calls - if call["transactionHash"] == hash + if call.transaction_hash == hash ] diff --git a/schemas/utils.py b/schemas/utils.py new file mode 100644 index 0000000..8a57059 --- /dev/null +++ b/schemas/utils.py @@ -0,0 +1,22 @@ +import json + +from pydantic import BaseModel + + +def to_camel(string: str) -> str: + return ''.join( + word.capitalize() if i > 0 else word + for i, word in enumerate(string.split('_')) + ) + + +class CamelModel(BaseModel): + """BaseModel that translates from camelCase to snake_case""" + + class Config: + alias_generator = to_camel + allow_population_by_field_name = True + + +def to_original_json_dict(model: BaseModel) -> dict: + return json.loads(model.json(by_alias=True, exclude_unset=True)) From d3982ba59b7c5891c5cb5a3b47521d95370efb98 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 12 Jul 2021 15:20:53 -0400 Subject: [PATCH 2/3] Include json_encoders in the CamelModel --- schemas/.utils.py.swp | Bin 0 -> 12288 bytes schemas/blocks.py | 12 ++---------- schemas/utils.py | 22 +++++++++++++++++----- 3 files changed, 19 insertions(+), 15 deletions(-) create mode 100644 schemas/.utils.py.swp diff --git a/schemas/.utils.py.swp b/schemas/.utils.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..fb8d39c29657a8a310307626e14e0823c45a2af4 GIT binary patch literal 12288 zcmeI2zi$&U6vtggL}>+df22b$QgbP-gixebqGf=gq60t1!*}N-ZhiLY>}#7MU;`Fl zV&EU(4`4uIWoG3sU?@mP9biS``FcqyD(Vb8OW*inKfmYi$6lgbJb&lL8e9n%7`AD~ z_FsOz_xi(dVrF-Osd7{J;gi}LPgacCY_AuleK}Mn;p*5W4}HQ;UYLF-<5G9}(hbTu zOib4CGh4QkiLJKOHW!I&=fjRo1|s9OQ#z?^m=BN5N`pjz2%JcubiMYKNw&CfY2L@q zHZH>XbGJ`iMgc^C2oM1xKm>>Y5g-CYfC&6M1YADB9-*C6wWUVgPyE)`U#TMkM1Tko z0U|&IhyW2F0z`la5CI}U1pYz-JY{U=3}f$5{{R2c@Bi;pjD16WM7>2lMLj`1My05A z)GTWMEMs3#pHX|LH>eDyP@AZmsA<#`>c=EwpHS~nuTU>g&r#1%IjVySQKwOR^`0)^ zGc_VW1c(3;AOb{y2oM1xKm`7C0#G-ul-Z7Qlb4D+X>=41qn;EhjWo{$EWxd!6znis zavSx976o^vz-ecq1OrtSk|=I1tQy^u{ot;M7uQTGRCqAZ>ar@R(P*r2E2=4Q1Ma{T zTw5$JEc6PKL6vY7V=y~d%{K+sumTsdx`MA*B?!i;eERV|AB>*>+jX(CGQ{j%P&|uM z4udWX!hzTcT60GNuR2%AxOC#0Ok5XI{5%ZUut-Pf1T7U7o!_sP{%ZM;1_xZH3f*lZ zlBR`HTTw`^&AVODOJ{GrW( z*hy5GipbY<)mW>=Qql8=W(wJtnyaX?<((cR>cSyWXfIa`Mr2!KSUo&@IS1RONI_~4 zy3G8I5P?O2x*ytHNf$JuCKiOc`QR|&xpdyS2wEU}AYmEiK`1gXnyia_wX?3K{!8n8xc7 R`%5#L9$g~F>XaTF`w1v4Q5gUL literal 0 HcmV?d00001 diff --git a/schemas/blocks.py b/schemas/blocks.py index fb71d97..18e41b1 100644 --- a/schemas/blocks.py +++ b/schemas/blocks.py @@ -2,11 +2,9 @@ import json from enum import Enum from typing import Dict, List, Optional -from hexbytes import HexBytes from pydantic import BaseModel -from web3.datastructures import AttributeDict -from .utils import CamelModel +from .utils import CamelModel, Web3Model class BlockCallType(Enum): @@ -30,7 +28,7 @@ class BlockCall(CamelModel): error: Optional[str] -class Block(BaseModel): +class Block(Web3Model): block_number: int calls: List[BlockCall] data: dict @@ -39,12 +37,6 @@ class Block(BaseModel): transaction_hashes: List[str] txs_gas_data: Dict[str, dict] - class Config: - json_encoders = { - AttributeDict: dict, - HexBytes: lambda h: h.hex(), - } - def get_filtered_calls(self, hash: str) -> List[BlockCall]: return [ call for call in self.calls diff --git a/schemas/utils.py b/schemas/utils.py index 8a57059..a3cb04b 100644 --- a/schemas/utils.py +++ b/schemas/utils.py @@ -1,6 +1,8 @@ import json +from hexbytes import HexBytes from pydantic import BaseModel +from web3.datastructures import AttributeDict def to_camel(string: str) -> str: @@ -10,13 +12,23 @@ def to_camel(string: str) -> str: ) +def to_original_json_dict(model: BaseModel) -> dict: + return json.loads(model.json(by_alias=True, exclude_unset=True)) + + +class Web3Model(BaseModel): + """BaseModel that handles web3's unserializable objects""" + + class Config: + json_encoders = { + AttributeDict: dict, + HexBytes: lambda h: h.hex(), + } + + class CamelModel(BaseModel): """BaseModel that translates from camelCase to snake_case""" - class Config: + class Config(Web3Model.Config): alias_generator = to_camel allow_population_by_field_name = True - - -def to_original_json_dict(model: BaseModel) -> dict: - return json.loads(model.json(by_alias=True, exclude_unset=True)) From de0853f50d94fcc49d79185f3621f2a26071e5f7 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 12 Jul 2021 15:25:48 -0400 Subject: [PATCH 3/3] Remove swp files --- .gitignore | 4 +++- schemas/.utils.py.swp | Bin 12288 -> 0 bytes 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 schemas/.utils.py.swp diff --git a/.gitignore b/.gitignore index fcc9a74..6cc5bdc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ # venv and test cache files env/ -__pycache__ \ No newline at end of file +__pycache__ + +*.swp diff --git a/schemas/.utils.py.swp b/schemas/.utils.py.swp deleted file mode 100644 index fb8d39c29657a8a310307626e14e0823c45a2af4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2zi$&U6vtggL}>+df22b$QgbP-gixebqGf=gq60t1!*}N-ZhiLY>}#7MU;`Fl zV&EU(4`4uIWoG3sU?@mP9biS``FcqyD(Vb8OW*inKfmYi$6lgbJb&lL8e9n%7`AD~ z_FsOz_xi(dVrF-Osd7{J;gi}LPgacCY_AuleK}Mn;p*5W4}HQ;UYLF-<5G9}(hbTu zOib4CGh4QkiLJKOHW!I&=fjRo1|s9OQ#z?^m=BN5N`pjz2%JcubiMYKNw&CfY2L@q zHZH>XbGJ`iMgc^C2oM1xKm>>Y5g-CYfC&6M1YADB9-*C6wWUVgPyE)`U#TMkM1Tko z0U|&IhyW2F0z`la5CI}U1pYz-JY{U=3}f$5{{R2c@Bi;pjD16WM7>2lMLj`1My05A z)GTWMEMs3#pHX|LH>eDyP@AZmsA<#`>c=EwpHS~nuTU>g&r#1%IjVySQKwOR^`0)^ zGc_VW1c(3;AOb{y2oM1xKm`7C0#G-ul-Z7Qlb4D+X>=41qn;EhjWo{$EWxd!6znis zavSx976o^vz-ecq1OrtSk|=I1tQy^u{ot;M7uQTGRCqAZ>ar@R(P*r2E2=4Q1Ma{T zTw5$JEc6PKL6vY7V=y~d%{K+sumTsdx`MA*B?!i;eERV|AB>*>+jX(CGQ{j%P&|uM z4udWX!hzTcT60GNuR2%AxOC#0Ok5XI{5%ZUut-Pf1T7U7o!_sP{%ZM;1_xZH3f*lZ zlBR`HTTw`^&AVODOJ{GrW( z*hy5GipbY<)mW>=Qql8=W(wJtnyaX?<((cR>cSyWXfIa`Mr2!KSUo&@IS1RONI_~4 zy3G8I5P?O2x*ytHNf$JuCKiOc`QR|&xpdyS2wEU}AYmEiK`1gXnyia_wX?3K{!8n8xc7 R`%5#L9$g~F>XaTF`w1v4Q5gUL