inital tokenflow/tests

.gitignore

@@ -0,0 +1,3 @@
+# venv and test cache files
+env/
+__pycache__

README.md

@@ -0,0 +1,19 @@
+
+### Local setup
+
+Requirements:
+
+* python3 and pip3
+
+Instructions: 
+
+* Setup a virtual enviroment to manage dependencies (optional)  
+    * `python3 -m venv env`
+* Activate it
+    * `. env/bin/activate` (exit with `deactivate`)
+* web3 build-related dependencies (on Ubuntu 20.04)
+    * `sudo apt-get install libevent-dev libpython3.8-dev python3.8-dev libssl-dev`
+* Install python libraries
+    * `pip3 install -r requirements.txt`
+* Run tests for token flow
+    * `python -m unittest tests/tokenflow_test.py`

block.py

@@ -5,13 +5,14 @@ import json
 cache_directoty = './cache'
 
 class BlockData:
-    def __init__(self, block_number, data, receipts, calls, logs) -> None:
+    def __init__(self, block_number, data, receipts, calls, logs, txs_gas_data) -> None:
         self.block_number = block_number
         self.data = data
         self.receipts = receipts
         self.calls = calls
         self.logs = logs
         self.transaction_hashes = self.get_transaction_hashes()
+        self.txs_gas_data = txs_gas_data
         pass
     
     ## Gets a list of unique transasction hashes in the calls of this block
@@ -34,7 +35,7 @@ class BlockData:
     ## Writes this object to a JSON file for loading later
     def writeJSON(self):
         json_data = self.toJSON()
-        cache_file = '{cacheDirectory}/{blockNumber}.json'.format(cacheDirectory=cache_directoty, blockNumber=self.block_number)
+        cache_file = '{cacheDirectory}/{blockNumber}-new.json'.format(cacheDirectory=cache_directoty, blockNumber=self.block_number)
         file_exists = Path(cache_file).is_file()
         if file_exists:
             f = open(cache_file, "w")
@@ -60,7 +61,7 @@ class BlockData:
 ## Note that you need to pass in the provider, not the web3 wrapped provider object!
 ## This is because only the provider allows you to make json rpc requests
 def createFromBlockNumber(block_number, base_provider):
-    cache_file = '{cacheDirectory}/{blockNumber}.json'.format(cacheDirectory=cache_directoty, blockNumber=block_number)
+    cache_file = '{cacheDirectory}/{blockNumber}-new.json'.format(cacheDirectory=cache_directoty, blockNumber=block_number)
     
     ## Check to see if the data already exists in the cache
     ## if it exists load the data from cache
@@ -69,7 +70,7 @@ def createFromBlockNumber(block_number, base_provider):
         print(('Cache for block {block_number} exists, loading data from cache').format(block_number=block_number))
         block_file = open(cache_file)
         block_json = json.load(block_file)
-        block = BlockData(block_number, block_json['data'], block_json['receipts'], block_json['calls'], block_json['logs'])
+        block = BlockData(block_number, block_json['data'], block_json['receipts'], block_json['calls'], block_json['logs'], block_json['txs_gas_data'])
         return block
     else:
         w3 = Web3(base_provider)
@@ -89,9 +90,21 @@ def createFromBlockNumber(block_number, base_provider):
         ## Get the logs
         block_hash = (block_data.hash).hex()
         block_logs = w3.eth.get_logs({'blockHash': block_hash})
+
+        ## Get gas used by individual txs and store them too
+        txs_gas_data = {}
+        for transaction in block_data['transactions']:
+            tx_hash = (transaction.hash).hex()
+            tx_data = w3.eth.get_transaction(tx_hash)
+            tx_receipt = w3.eth.wait_for_transaction_receipt(tx_hash)
+            txs_gas_data[tx_hash] = {
+                'gasUsed': tx_receipt['gasUsed'], # fix: why does this return 0 for certain txs?
+                'gasPrice': tx_data['gasPrice'],
+                'netFeePaid': tx_data['gasPrice'] * tx_receipt['gasUsed']
+            }
         
         ## Create a new object
-        block = BlockData(block_number, block_data, block_receipts_raw, block_calls, block_logs)
+        block = BlockData(block_number, block_data, block_receipts_raw, block_calls, block_logs, txs_gas_data)
         
         ## Write the result to a JSON file for loading in the future
         block.writeJSON()

requirements.txt

@@ -0,0 +1,3 @@
+web3==5.20.1
+hexbytes==0.2.1
+argparse==1.4.0

tests/liquidation_test.py

@@ -0,0 +1,19 @@
+import unittest
+import inspector_compound
+import inspector_aave
+
+class TestLiquidations (unittest.TestCase):
+    def test_compound_liquidation(self):
+        tx_hash = "0x0ec6d5044a47feb3ceb647bf7ea4ffc87d09244d629eeced82ba17ec66605012"
+        block_no = 11338848
+        res = inspector_compound.get_profit(tx_hash, block_no)
+        # self.assertEqual(res['profit'], 0)
+    def test_aave_liquidation(self):
+        tx_hash = "0xc8d2501d28800b1557eb64c5d0e08fd6070c15b6c04c39ca05631f641d19ffb2"
+        block_no = 10803840
+        res = inspector_aave.get_profit(tx_hash, block_no)
+        # self.assertEqual(res['profit'], 0)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/tokenflow_test.py

@@ -0,0 +1,27 @@
+import unittest
+import tokenflow
+
+class TestTokenFlow (unittest.TestCase):
+    def test_simple_arb(self):
+        tx_hash = "0x4121ce805d33e952b2e6103a5024f70c118432fd0370128d6d7845f9b2987922"
+        block_no = 11930296
+        res = tokenflow.run_tokenflow(tx_hash, block_no)
+        self.assertEqual(res['ether_flows'], [3547869861992962562, 3499859860420296704])
+        self.assertEqual(res['dollar_flows'], [0,0])
+    
+    def test_arb_with_stable_flow(self):
+        tx_hash = "0x496836e0bd1520388e36c79d587a31d4b3306e4f25352164178ca0667c7f9c29"
+        block_no = 11935012
+        res = tokenflow.run_tokenflow(tx_hash, block_no)
+        self.assertEqual(res['ether_flows'], [597044987302243493, 562445964778930176])
+        self.assertEqual(res['dollar_flows'], [871839781,871839781])
+    
+    def test_complex_cross_arb(self):
+        tx_hash = "0x5ab21bfba50ad3993528c2828c63e311aafe93b40ee934790e545e150cb6ca73"
+        block_no = 11931272
+        res = tokenflow.run_tokenflow(tx_hash, block_no)
+        self.assertEqual(res['ether_flows'], [3636400213125714803, 3559576672903063566])
+        self.assertEqual(res['dollar_flows'], [0,0])       
+
+if __name__ == '__main__':
+    unittest.main()

tokenflow.py

@@ -0,0 +1,239 @@
+# from web3 import Web3, HTTPProvider
+from pathlib import Path
+import json
+import configparser
+
+config = configparser.ConfigParser()
+config.read('./utils/config.ini')
+rpc_url = config['RPC']['Endpoint']
+weth_address = config['ADDRESSES']['WETH']
+# w3 = Web3(HTTPProvider(rpc_url))
+
+cache_directory = './cache'
+
+def get_tx_traces(txHash, blockNo):
+    # block_calls = w3.parity.trace_block(10803840)
+    cache_file = '{cacheDirectory}/{blockNumber}-new.json'.format(cacheDirectory=cache_directory, blockNumber=blockNo)
+    file_exists = Path(cache_file).is_file()
+    
+    tx_traces = []
+    # if have the traces cached
+    if(file_exists):
+        block_file = open(cache_file)
+        block_json = json.load(block_file)
+        for call in block_json['calls']:
+            if call['transactionHash'] == txHash:
+                tx_traces.append(call)
+        block_file.close()
+    else:
+        # todo, fetch and cache traces that don't exist
+        # depending on the best way to call block.py from here 
+        print("traces do not exist")
+    
+    return(tx_traces)
+
+def is_stablecoin_address(address):
+    # to look for stablecoin inflow/outflows
+    stablecoin_addresses = [
+        "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", # USDC
+        "0xdac17f958d2ee523a2206206994597c13d831ec7", # USDT
+        "0x6b175474e89094c44da98b954eedeac495271d0f", # DAI
+        "0x0000000000085d4780b73119b644ae5ecd22b376", # TUSD
+        "0x4fabb145d64652a948d72533023f6e7a623c7c53", # BUSD
+        "0x8e870d67f660d95d5be530380d0ec0bd388289e1", # PAX
+        "0x956F47F50A910163D8BF957Cf5846D573E7f87CA", # FEI
+        "0x853d955aCEf822Db058eb8505911ED77F175b99e", # FRAX
+        "0xBC6DA0FE9aD5f3b0d58160288917AA56653660E9", # alUSD
+        "0x57Ab1ec28D129707052df4dF418D58a2D46d5f51", # sUSD
+        "0x5f98805A4E8be255a32880FDeC7F6728C6568bA0", # lUSD
+        "0x674C6Ad92Fd080e4004b2312b45f796a192D27a0", # USDN
+    ]
+    return address in stablecoin_addresses
+
+def is_known_router_address(address):
+    # to exclude known router addresses from token flow analysis
+    known_router_addresses = [    
+        "0x3D71d79C224998E608d03C5Ec9B405E7a38505F0", # keeper dao, whitelists extraction
+        "0x11111254369792b2Ca5d084aB5eEA397cA8fa48B", # 1inch v1 router
+        "0x111111125434b319222cdbf8c261674adb56f3ae", # 1inch v2 router
+        "0x11111112542d85b3ef69ae05771c2dccff4faa26", # 1inch v3 router
+        "0xa356867fdcea8e71aeaf87805808803806231fdc", # DODO
+        "0xdef1c0ded9bec7f1a1670819833240f027b25eff", # 0x proxy
+        "0x90f765f63e7dc5ae97d6c576bf693fb6af41c129", # Set Trade
+        "0x7113dd99c79aff93d54cfa4b2885576535a132de", # Totle exchange
+        "0x9509665d015bfe3c77aa5ad6ca20c8afa1d98989", # Paraswap
+        "0x86969d29F5fd327E1009bA66072BE22DB6017cC6", # Paraswap v2
+        "0xf90e98f3d8dce44632e5020abf2e122e0f99dfab", # Paraswap v3
+        "0x57805e5a227937bac2b0fdacaa30413ddac6b8e1", # Furucombo
+        "0x17e8ca1b4798b97602895f63206afcd1fc90ca5f", # Furucombo proxy
+        "0x881d40237659c251811cec9c364ef91dc08d300c", # Metamask swap
+        "0x745daa146934b27e3f0b6bff1a6e36b9b90fb131", # DEX.ag
+        "0xb2be281e8b11b47fec825973fc8bb95332022a54", # Zerion SDK
+        "0x7a250d5630B4cF539739dF2C5dAcb4c659F2488D", # UniswapV2Router02
+        "0xd9e1cE17f2641f24aE83637ab66a2cca9C378B9F", # SushiswapV2Router02
+        "0xE592427A0AEce92De3Edee1F18E0157C05861564", # Uniswap v3 router
+        "0x3E66B66Fd1d0b02fDa6C811Da9E0547970DB2f21", # Balance exchange proxy
+        "0x1bD435F3C054b6e901B7b108a0ab7617C808677b", # Paraswap v4
+        "0xC011a73ee8576Fb46F5E1c5751cA3B9Fe0af2a6F" # SNX proxy synth issuer
+    ]
+    return address in known_router_addresses
+
+# we're interested in the to address to run token flow on it as well
+def get_tx_to_address(txHash, blockNo):
+    cache_file = '{cacheDirectory}/{blockNumber}-new.json'.format(cacheDirectory=cache_directory, blockNumber=blockNo)
+    block_file = open(cache_file)
+    block_json = json.load(block_file)
+    for receipt in block_json['receipts']['result']:
+        if receipt['transactionHash'] == txHash:
+            block_file.close()
+            return receipt['to']
+    
+
+def get_tx_proxies(tx_traces, to_address):
+    proxies = []
+    for trace in tx_traces:
+        if(trace['type'] == 'call' and trace['action']['callType'] == 'delegatecall' and trace['action']['from'] == to_address):
+            proxies.append(trace['action']['to'])
+    return(proxies)
+
+def get_net_gas_used(txHash, blockNo):
+    cache_file = '{cacheDirectory}/{blockNumber}.json'.format(cacheDirectory=cache_directory, blockNumber=blockNo)
+    block_file = open(cache_file)
+    block_json = json.load(block_file)
+    gas_used = 0
+    for trace in block_json['calls']:
+        if trace['transactionHash'] == txHash:
+            gas_used = gas_used + int(trace['result']['gasUsed'],16)
+    print(gas_used)
+
+def get_ether_flows(tx_traces, addresses_to_check):
+    eth_inflow = 0
+    eth_outflow = 0
+
+    for trace in tx_traces:
+        if(trace['type'] == 'call'):
+            value = int(trace['action']['value'], 16) # converting from 0x prefix to decimal
+            # ETH_GET
+            if(trace['action']['callType'] != 'delegatecall' and trace['action']['from'] != weth_address and value > 0 and trace['action']['to'] in addresses_to_check):
+                eth_inflow = eth_inflow + value
+
+            # ETH_GIVE
+            if(trace['action']['callType'] != 'delegatecall' and trace['action']['to'] != weth_address and value > 0 and trace['action']['from'] in addresses_to_check):
+                eth_outflow = eth_outflow + value
+
+            if(trace['action']['to'] == weth_address):
+                # WETH_GET1 & WETH_GET2 (to account for both 'transfer' and 'transferFrom' methods)
+                # WETH_GIVE1 & WETH_GIVE2
+
+                # transfer(address to,uint256 value) with args
+                if(len(trace['action']['input']) == 138):
+                    if(trace['action']['input'][2:10] == "a9059cbb"):
+                        transfer_to = '0x' + trace['action']['input'][34:74]
+                        transfer_value = int('0x' + trace['action']['input'][74:138], 16)
+                        if(transfer_to in addresses_to_check):
+                            eth_inflow = eth_inflow + transfer_value
+                        elif(trace['action']['from'] in addresses_to_check):
+                            eth_outflow = eth_outflow + transfer_value
+
+                # transferFrom(address from,address to,uint256 value )
+                if(len(trace['action']['input']) == 202):
+                    if(trace['action']['input'][2:10] == "23b872dd"):
+                        transfer_from = '0x' + trace['action']['input'][34:74]
+                        transfer_to = '0x' + trace['action']['input'][98:138]
+                        transfer_value = int('0x' + trace['action']['input'][138:202], 16)
+                        if(transfer_to in addresses_to_check):
+                            eth_inflow = eth_inflow + transfer_value
+                        elif(transfer_from in addresses_to_check):
+                            eth_outflow = eth_outflow + transfer_value
+        
+        if(trace['type'] == 'suicide'):
+            if(trace['action']['refundAddress'] in addresses_to_check):
+                refund_value = int('0x' + trace['action']['balance'], 16)
+                eth_inflow = eth_inflow + refund_value
+
+    return [eth_inflow, eth_outflow]
+
+def get_dollar_flows(tx_traces, addresses_to_check):
+    dollar_inflow = 0
+    dollar_outflow = 0
+    for trace in tx_traces:
+        if(trace['type'] == 'call' and is_stablecoin_address(trace['action']['to'])):
+            value = int(trace['action']['value'], 16) # converting from 0x prefix to decimal
+
+            # USD_GET1 & USD_GET2 (to account for both 'transfer' and 'transferFrom' methods)
+            # USD_GIVE1 & USD_GIVE2
+
+            # transfer(address to,uint256 value) with args
+            if(len(trace['action']['input']) == 138):
+                if(trace['action']['input'][2:10] == "a9059cbb"):
+                    transfer_to = '0x' + trace['action']['input'][34:74]
+                    transfer_value = int('0x' + trace['action']['input'][74:138], 16)
+                    if(transfer_to in addresses_to_check):
+                        dollar_inflow = dollar_inflow + transfer_value
+                    elif(trace['action']['from'] in addresses_to_check):
+                        dollar_outflow = dollar_outflow + transfer_value
+
+            # transferFrom(address from,address to,uint256 value )
+            if(len(trace['action']['input']) == 202):
+                if(trace['action']['input'][2:10] == "23b872dd"):
+                    transfer_from = '0x' + trace['action']['input'][34:74]
+                    transfer_to = '0x' + trace['action']['input'][98:138]
+                    transfer_value = int('0x' + trace['action']['input'][138:202], 16)
+                    if(transfer_to in addresses_to_check):
+                        dollar_inflow = dollar_inflow + transfer_value
+                    elif(transfer_from in addresses_to_check):
+                        dollar_outflow = dollar_outflow + transfer_value
+    return [dollar_inflow, dollar_outflow]
+
+def run_tokenflow(txHash, blockNo):
+    tx_traces = get_tx_traces(txHash, blockNo)
+    to_address = get_tx_to_address(txHash, blockNo)
+    addresses_to_check = []
+
+    # check for proxies, add them to addresses to check
+    proxies = get_tx_proxies(tx_traces, to_address)
+    for proxy in proxies:
+        addresses_to_check.append(proxy.lower())
+    
+    # check if the 'to' field is a known aggregator/router
+    # if not, add to relevant addresses to run TF on
+    if(not is_known_router_address(to_address)):
+        addresses_to_check.append(to_address.lower()) # traces need lowercase addresses to match
+    
+    ether_flows = get_ether_flows(tx_traces, addresses_to_check)
+    dollar_flows = get_dollar_flows(tx_traces, addresses_to_check)
+    # print(addresses_to_check)
+    # print('net eth flow', ether_flows[0] - ether_flows[1])
+    # print('net dollar flow', dollar_flows )
+    return {
+        'ether_flows': ether_flows,
+        'dollar_flows': dollar_flows
+    }
+
+
+# note: not the gas set by user, only gas consumed upon execution
+def get_gas_used_by_tx(txHash):
+    #tx_receipt = w3.eth.getTransactionReceipt(txHash)
+    return(tx_receipt['gasUsed'])
+
+
+# tx_traces = get_tx_traces('0x4121ce805d33e952b2e6103a5024f70c118432fd0370128d6d7845f9b2987922', 11930296)
+# print(tx_traces)
+
+# print(type(known_router_addresses))
+# print(is_stablecoin_address("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48"))
+
+# run_tokenflow("0x4121ce805d33e952b2e6103a5024f70c118432fd0370128d6d7845f9b2987922", 11930296)
+
+# delegate call test
+# run_tokenflow("0x9007b339c81de366cd53539edc15c86ffc87542c65f374c0d4d1f8823a3ccf60", 12051659)
+
+# stable flow test
+# res  = run_tokenflow("0x496836e0bd1520388e36c79d587a31d4b3306e4f25352164178ca0667c7f9c29", 11935012)
+# print(res)
+
+# complex arb test
+# res = run_tokenflow("0x5ab21bfba50ad3993528c2828c63e311aafe93b40ee934790e545e150cb6ca73", 11931272)
+# print(res)
+
+# get_gas_used_by_tx("0x4121ce805d33e952b2e6103a5024f70c118432fd0370128d6d7845f9b2987922")