Merge pull request #22 from lukevs/add-pydantic-calls-2

Add BlockCall class

.gitignore

@@ -1,3 +1,5 @@
 # venv and test cache files
 env/
-__pycache__
+__pycache__
+
+*.swp

block.py

@@ -4,100 +4,103 @@ from typing import List
 
 from web3 import Web3
 
-from schemas import Block
+from schemas import Block, BlockCall, BlockCallType
 
 
 cache_directory = './cache'
 
 
-def get_transaction_hashes(calls: List[dict]) -> List[str]:
-    result = []
-
-    for call in calls:
-        if call['type'] != 'reward':
-            if call['transactionHash'] in result:
-                continue
-            else:
-                result.append(call['transactionHash'])
-    
-    return result
-
-
-def write_json(block: Block):
-    cache_path = _get_cache_path(block.block_number)
-    write_mode = "w" if cache_path.is_file() else "x"
-
-    with open(cache_path, mode=write_mode) as cache_file:
-        cache_file.write(block.json(sort_keys=True, indent=4))
-
-
 ## Creates a block object, either from the cache or from the chain itself
 ## Note that you need to pass in the provider, not the web3 wrapped provider object!
 ## This is because only the provider allows you to make json rpc requests
 def createFromBlockNumber(block_number: int, base_provider) -> Block:
     cache_path = _get_cache_path(block_number)
-    
-    ## Check to see if the data already exists in the cache
-    ## if it exists load the data from cache
-    ## If not then get the data from the chain and save it to the cache
+
     if (cache_path.is_file()):
         print(
             f'Cache for block {block_number} exists, ' \
             'loading data from cache'
         )
 
-        block = Block.parse_file(cache_path)
-        return block
+        return Block.parse_file(cache_path)
     else:
-        w3 = Web3(base_provider)
-        print(("Cache for block {block_number} did not exist, getting data").format(block_number=block_number))
-        
-        ## Get block data
-        block_data = w3.eth.get_block(block_number, True)
-        
-        ## Get the block receipts
-        ## TODO: evaluate whether or not this is sufficient or if gas used needs to be converted to a proper big number.
-        ## In inspect-ts it needed to be converted
-        block_receipts_raw = base_provider.make_request("eth_getBlockReceipts", [block_number])
-
-        ## Trace the whole block, return those calls
-        block_calls = w3.parity.trace_block(block_number)
-        
-        ## Get the logs
-        block_hash = (block_data.hash).hex()
-        block_logs = w3.eth.get_logs({'blockHash': block_hash})
-
-        ## Get gas used by individual txs and store them too
-        txs_gas_data = {}
-        for transaction in block_data['transactions']:
-            tx_hash = (transaction.hash).hex()
-            tx_data = w3.eth.get_transaction(tx_hash)
-            tx_receipt = w3.eth.wait_for_transaction_receipt(tx_hash)
-            txs_gas_data[tx_hash] = {
-                'gasUsed': tx_receipt['gasUsed'], # fix: why does this return 0 for certain txs?
-                'gasPrice': tx_data['gasPrice'],
-                'netFeePaid': tx_data['gasPrice'] * tx_receipt['gasUsed']
-            }
-        
-        transaction_hashes = get_transaction_hashes(block_calls)
-
-        ## Create a new object
-        block = Block(
-            block_number=block_number,
-            data=block_data,
-            receipts=block_receipts_raw,
-            calls=block_calls,
-            logs=block_logs,
-            transaction_hashes=transaction_hashes,
-            txs_gas_data=txs_gas_data,
+        print(
+           f"Cache for block {block_number} did not exist, getting data"
         )
-        
-        ## Write the result to a JSON file for loading in the future
-        write_json(block)
+
+        w3 = Web3(base_provider)
+        block = fetch_block(w3, base_provider, block_number)
+
+        cache_block(cache_path, block)
 
         return block
 
 
+def fetch_block(w3, base_provider, block_number: int) -> Block:
+    ## Get block data
+    block_data = w3.eth.get_block(block_number, True)
+    
+    ## Get the block receipts
+    ## TODO: evaluate whether or not this is sufficient or if gas used needs to be converted to a proper big number.
+    ## In inspect-ts it needed to be converted
+    block_receipts_raw = base_provider.make_request("eth_getBlockReceipts", [block_number])
+
+    ## Trace the whole block, return those calls
+    block_calls_json = w3.parity.trace_block(block_number)
+    block_calls = [
+        BlockCall(**call_json)
+        for call_json in block_calls_json
+    ]
+    
+    ## Get the logs
+    block_hash = (block_data.hash).hex()
+    block_logs = w3.eth.get_logs({'blockHash': block_hash})
+
+    ## Get gas used by individual txs and store them too
+    txs_gas_data = {}
+
+    for transaction in block_data['transactions']:
+        tx_hash = (transaction.hash).hex()
+        tx_data = w3.eth.get_transaction(tx_hash)
+        tx_receipt = w3.eth.wait_for_transaction_receipt(tx_hash)
+        txs_gas_data[tx_hash] = {
+            'gasUsed': tx_receipt['gasUsed'], # fix: why does this return 0 for certain txs?
+            'gasPrice': tx_data['gasPrice'],
+            'netFeePaid': tx_data['gasPrice'] * tx_receipt['gasUsed']
+        }
+    
+    transaction_hashes = get_transaction_hashes(block_calls)
+
+    ## Create a new object
+    return Block(
+        block_number=block_number,
+        data=block_data,
+        receipts=block_receipts_raw,
+        calls=block_calls,
+        logs=block_logs,
+        transaction_hashes=transaction_hashes,
+        txs_gas_data=txs_gas_data,
+    )
+
+
+def get_transaction_hashes(calls: List[BlockCall]) -> List[str]:
+    result = []
+
+    for call in calls:
+        if call.type != BlockCallType.reward:
+            if call.transaction_hash not in result:
+                result.append(call.transaction_hash)
+
+    return result
+
+
+def cache_block(cache_path: Path, block: Block):
+    write_mode = "w" if cache_path.is_file() else "x"
+
+    with open(cache_path, mode=write_mode) as cache_file:
+        cache_file.write(block.json())
+
+
 def _get_cache_path(block_number: int) -> Path:
     cache_directory_path = Path(cache_directory)
     return cache_directory_path / f"{block_number}-new.json"

processor.py

@@ -1,3 +1,5 @@
+from schemas.utils import to_original_json_dict
+
 
 class Processor:
     def __init__(self, base_provider, inspectors) -> None:
@@ -7,7 +9,10 @@ class Processor:
     def get_transaction_evaluations(self, block_data):
         for transaction_hash in block_data.transaction_hashes:
             calls = block_data.get_filtered_calls(transaction_hash)
+            calls_json = [
+                to_original_json_dict(call)
+                for call in calls
+            ]
 
             for inspector in self.inspectors:
-                inspector.inspect(calls)
-                # print(calls)
+                inspector.inspect(calls_json)

schemas/__init__.py

@@ -1 +1 @@
-from .blocks import Block
+from .blocks import Block, BlockCall, BlockCallType

schemas/blocks.py

@@ -1,28 +1,44 @@
 import json
+from enum import Enum
 from typing import Dict, List, Optional
 
-from hexbytes import HexBytes
 from pydantic import BaseModel
-from web3.datastructures import AttributeDict
+
+from .utils import CamelModel, Web3Model
 
 
-class Block(BaseModel):
+class BlockCallType(Enum):
+    call = "call"
+    create = "create"
+    delegate_call = "delegateCall"
+    reward = "reward"
+    suicide = "suicide"
+
+
+class BlockCall(CamelModel):
+    action: dict
+    block_hash: str
     block_number: int
-    calls: List[dict]
+    result: Optional[dict]
+    subtraces: int
+    trace_address: List[int]
+    transaction_hash: Optional[str]
+    transaction_position: Optional[int]
+    type: BlockCallType
+    error: Optional[str]
+
+
+class Block(Web3Model):
+    block_number: int
+    calls: List[BlockCall]
     data: dict
     logs: List[dict]
     receipts: dict
     transaction_hashes: List[str]
     txs_gas_data: Dict[str, dict]
 
-    class Config:
-        json_encoders = {
-            AttributeDict: dict,
-            HexBytes: lambda h: h.hex(),
-        }
-
-    def get_filtered_calls(self, hash: str) -> List[dict]:
+    def get_filtered_calls(self, hash: str) -> List[BlockCall]:
         return [
             call for call in self.calls
-            if call["transactionHash"] == hash
+            if call.transaction_hash == hash
         ]

schemas/utils.py

@@ -0,0 +1,34 @@
+import json
+
+from hexbytes import HexBytes
+from pydantic import BaseModel
+from web3.datastructures import AttributeDict
+
+
+def to_camel(string: str) -> str:
+    return ''.join(
+        word.capitalize() if i > 0 else word
+        for i, word in enumerate(string.split('_'))
+    )
+
+
+def to_original_json_dict(model: BaseModel) -> dict:
+    return json.loads(model.json(by_alias=True, exclude_unset=True))
+
+
+class Web3Model(BaseModel):
+    """BaseModel that handles web3's unserializable objects"""
+
+    class Config:
+        json_encoders = {
+            AttributeDict: dict,
+            HexBytes: lambda h: h.hex(),
+        }
+
+
+class CamelModel(BaseModel):
+    """BaseModel that translates from camelCase to snake_case"""
+
+    class Config(Web3Model.Config):
+        alias_generator = to_camel
+        allow_population_by_field_name = True