From 1d3e2cc000560dd23523a8ec4607d4b9fdb1eec4 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Sun, 11 Jul 2021 11:23:38 -0400 Subject: [PATCH 1/7] Add requirements.txt with web3 and pydantic --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4a11008..7b5d209 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ web3==5.20.1 hexbytes==0.2.1 -argparse==1.4.0 \ No newline at end of file +argparse==1.4.0 +pydantic==1.8.2 From e67922b5e69368976d431ff9eaadf810238dc8e1 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Sun, 11 Jul 2021 12:06:23 -0400 Subject: [PATCH 2/7] Fix typo in cache_directory --- block.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/block.py b/block.py index 6e45b69..ede2bf3 100644 --- a/block.py +++ b/block.py @@ -2,7 +2,7 @@ from web3 import Web3 from pathlib import Path import json -cache_directoty = './cache' +cache_directory = './cache' class BlockData: def __init__(self, block_number, data, receipts, calls, logs, txs_gas_data) -> None: @@ -13,7 +13,6 @@ class BlockData: self.logs = logs self.transaction_hashes = self.get_transaction_hashes() self.txs_gas_data = txs_gas_data - pass ## Gets a list of unique transasction hashes in the calls of this block def get_transaction_hashes(self): @@ -108,4 +107,4 @@ def createFromBlockNumber(block_number, base_provider): ## Write the result to a JSON file for loading in the future block.writeJSON() - return block \ No newline at end of file + return block From bfc138f740a18ca19e14ff6e9a0d584cf7544454 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 12 Jul 2021 08:21:03 -0400 Subject: [PATCH 3/7] Add Block schema --- schemas/__init__.py | 1 + schemas/blocks.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 schemas/__init__.py create mode 100644 schemas/blocks.py diff --git a/schemas/__init__.py b/schemas/__init__.py new file mode 100644 index 0000000..7e8c285 --- /dev/null +++ b/schemas/__init__.py @@ -0,0 +1 @@ +from .blocks import Block diff --git a/schemas/blocks.py b/schemas/blocks.py new file mode 100644 index 0000000..2c1e779 --- /dev/null +++ b/schemas/blocks.py @@ -0,0 +1,12 @@ +from typing import List, Optional + +from pydantic import BaseModel + + +class Block(BaseModel): + block_number: int + calls: List[dict] + data: dict + logs: List[dict] + receipts: dict + transaction_hashes: List[str] From 7d51dd34421740b2ef28744e1d2b60f88b4d65de Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 12 Jul 2021 08:58:28 -0400 Subject: [PATCH 4/7] Reimplement block using the pydantic object --- block.py | 111 ++++++++++++++++++++++------------------------ schemas/blocks.py | 9 +++- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/block.py b/block.py index ede2bf3..1b016b4 100644 --- a/block.py +++ b/block.py @@ -1,75 +1,52 @@ -from web3 import Web3 -from pathlib import Path import json +from pathlib import Path +from typing import List + +from web3 import Web3 + +from schemas import Block + cache_directory = './cache' -class BlockData: - def __init__(self, block_number, data, receipts, calls, logs, txs_gas_data) -> None: - self.block_number = block_number - self.data = data - self.receipts = receipts - self.calls = calls - self.logs = logs - self.transaction_hashes = self.get_transaction_hashes() - self.txs_gas_data = txs_gas_data - - ## Gets a list of unique transasction hashes in the calls of this block - def get_transaction_hashes(self): - result = [] - for call in self.calls: - if call['type'] != 'reward': - if call['transactionHash'] in result: - continue - else: - result.append(call['transactionHash']) - - return result - ## Makes a nicely formatted JSON object out of this data object. - def toJSON(self): - return json.dumps(self, default=lambda o: o.__dict__, - sort_keys=True, indent=4) - - ## Writes this object to a JSON file for loading later - def writeJSON(self): - json_data = self.toJSON() - cache_file = '{cacheDirectory}/{blockNumber}-new.json'.format(cacheDirectory=cache_directoty, blockNumber=self.block_number) - file_exists = Path(cache_file).is_file() - if file_exists: - f = open(cache_file, "w") - f.write(json_data) - f.close() - else: - f = open(cache_file, "x") - f.write(json_data) - f.close() - - ## Gets all the calls associated with a transaction hash - def get_filtered_calls(self, hash): - result = [] +def get_transaction_hashes(calls: List[dict]) -> List[str]: + result = [] - for call in self.calls: - if call['transactionHash'] == hash: - result.append(call) - - return result + for call in calls: + if call['type'] != 'reward': + if call['transactionHash'] in result: + continue + else: + result.append(call['transactionHash']) + + return result + + +def write_json(block: Block): + cache_path = _get_cache_path(block.block_number) + write_mode = "w" if cache_path.is_file() else "x" + + with open(cache_path, mode=write_mode) as cache_file: + cache_file.write(block.json()) ## Creates a block object, either from the cache or from the chain itself ## Note that you need to pass in the provider, not the web3 wrapped provider object! ## This is because only the provider allows you to make json rpc requests -def createFromBlockNumber(block_number, base_provider): - cache_file = '{cacheDirectory}/{blockNumber}-new.json'.format(cacheDirectory=cache_directoty, blockNumber=block_number) +def createFromBlockNumber(block_number: int, base_provider) -> Block: + cache_path = _get_cache_path(block_number) ## Check to see if the data already exists in the cache ## if it exists load the data from cache ## If not then get the data from the chain and save it to the cache - if (Path(cache_file).is_file()): - print(('Cache for block {block_number} exists, loading data from cache').format(block_number=block_number)) - block_file = open(cache_file) - block_json = json.load(block_file) - block = BlockData(block_number, block_json['data'], block_json['receipts'], block_json['calls'], block_json['logs'], block_json['txs_gas_data']) + if (cache_path.is_file()): + print( + f'Cache for block {block_number} exists, ' \ + 'loading data from cache' + ) + + block = Block.parse_file(cache_path) return block else: w3 = Web3(base_provider) @@ -102,9 +79,25 @@ def createFromBlockNumber(block_number, base_provider): 'netFeePaid': tx_data['gasPrice'] * tx_receipt['gasUsed'] } + transaction_hashes = get_transaction_hashes(block_calls) + ## Create a new object - block = BlockData(block_number, block_data, block_receipts_raw, block_calls, block_logs, txs_gas_data) + block = Block( + block_number=block_number, + data=block_data, + receipts=block_receipts_raw, + calls=block_calls, + logs=block_logs, + transaction_hashes=transaction_hashes, + txs_gas_data=txs_gas_data, + ) ## Write the result to a JSON file for loading in the future - block.writeJSON() + write_json(block) + return block + + +def _get_cache_path(block_number: int) -> Path: + cache_filepath = f"{cache_directory}/{block_number}-new.json" + return Path(cache_filepath) diff --git a/schemas/blocks.py b/schemas/blocks.py index 2c1e779..66851b9 100644 --- a/schemas/blocks.py +++ b/schemas/blocks.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Dict, List, Optional from pydantic import BaseModel @@ -10,3 +10,10 @@ class Block(BaseModel): logs: List[dict] receipts: dict transaction_hashes: List[str] + txs_gas_data: Dict[str, dict] + + def get_filtered_calls(self, hash: str) -> List[dict]: + return [ + call for call in self.calls + if call["transactionHash"] == hash + ] From 5e9c350b3882956f0ee3083114eb5b6fce60f3cf Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 12 Jul 2021 09:55:11 -0400 Subject: [PATCH 5/7] Slightly better implementation for block path --- block.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block.py b/block.py index 1b016b4..f0fe5e4 100644 --- a/block.py +++ b/block.py @@ -99,5 +99,5 @@ def createFromBlockNumber(block_number: int, base_provider) -> Block: def _get_cache_path(block_number: int) -> Path: - cache_filepath = f"{cache_directory}/{block_number}-new.json" - return Path(cache_filepath) + cache_directory_path = Path(cache_directory) + return cache_directory_path / f"{block_number}-new.json" From f0274fdfa84bc7a3c5296c56cba708e4a91cbfc8 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 12 Jul 2021 13:02:58 -0400 Subject: [PATCH 6/7] Add support for AttributeDict and HexBytes serialization --- schemas/blocks.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/schemas/blocks.py b/schemas/blocks.py index 66851b9..35aa340 100644 --- a/schemas/blocks.py +++ b/schemas/blocks.py @@ -1,6 +1,9 @@ +import json from typing import Dict, List, Optional +from hexbytes import HexBytes from pydantic import BaseModel +from web3.datastructures import AttributeDict class Block(BaseModel): @@ -12,6 +15,12 @@ class Block(BaseModel): transaction_hashes: List[str] txs_gas_data: Dict[str, dict] + class Config: + json_encoders = { + AttributeDict: dict, + HexBytes: lambda h: h.hex(), + } + def get_filtered_calls(self, hash: str) -> List[dict]: return [ call for call in self.calls From 14fc2396f376f47cf1c43c1b4fb9029861aef103 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 12 Jul 2021 13:06:19 -0400 Subject: [PATCH 7/7] Add back indent and sort_keys --- block.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block.py b/block.py index f0fe5e4..1694538 100644 --- a/block.py +++ b/block.py @@ -28,7 +28,7 @@ def write_json(block: Block): write_mode = "w" if cache_path.is_file() else "x" with open(cache_path, mode=write_mode) as cache_file: - cache_file.write(block.json()) + cache_file.write(block.json(sort_keys=True, indent=4)) ## Creates a block object, either from the cache or from the chain itself