Compound backfilling/removed network calls (#125)

* Removes `collateral_token_address` from both aave/comp for consistency

alembic/versions/b9fa1ecc9929_remove_liq_column.py

@@ -0,0 +1,27 @@
+"""Remove collateral_token_address column
+
+Revision ID: b9fa1ecc9929
+Revises: 04b76ab1d2af
+Create Date: 2021-12-01 23:32:40.574108
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "b9fa1ecc9929"
+down_revision = "04b76ab1d2af"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_column("liquidations", "collateral_token_address")
+
+
+def downgrade():
+    op.add_column(
+        "liquidations",
+        sa.Column("collateral_token_address", sa.String(256), nullable=False),
+    )

mev_inspect/aave_liquidations.py

@@ -66,7 +66,6 @@ def get_aave_liquidations(
             liquidations.append(
                 Liquidation(
                     liquidated_user=trace.inputs["_user"],
-                    collateral_token_address=trace.inputs["_collateral"],
                     debt_token_address=trace.inputs["_reserve"],
                     liquidator_user=liquidator,
                     debt_purchase_amount=trace.inputs["_purchaseAmount"],

mev_inspect/compound_liquidations.py

@@ -1,5 +1,4 @@
-from typing import Dict, List, Optional
+from typing import List, Optional
-from web3 import Web3
 
 from mev_inspect.traces import get_child_traces
 from mev_inspect.schemas.traces import (
@@ -9,44 +8,15 @@ from mev_inspect.schemas.traces import (
 )
 
 from mev_inspect.schemas.liquidations import Liquidation
-from mev_inspect.abi import get_raw_abi
-from mev_inspect.transfers import ETH_TOKEN_ADDRESS
 
 V2_COMPTROLLER_ADDRESS = "0x3d9819210A31b4961b30EF54bE2aeD79B9c9Cd3B"
 V2_C_ETHER = "0x4Ddc2D193948926D02f9B1fE9e1daa0718270ED5"
 CREAM_COMPTROLLER_ADDRESS = "0x3d5BC3c8d13dcB8bF317092d84783c2697AE9258"
 CREAM_CR_ETHER = "0xD06527D5e56A3495252A528C4987003b712860eE"
 
-# helper, only queried once in the beginning (inspect_block)
-def fetch_all_underlying_markets(w3: Web3, protocol: Protocol) -> Dict[str, str]:
-    if protocol == Protocol.compound_v2:
-        c_ether = V2_C_ETHER
-        address = V2_COMPTROLLER_ADDRESS
-    elif protocol == Protocol.cream:
-        c_ether = CREAM_CR_ETHER
-        address = CREAM_COMPTROLLER_ADDRESS
-    else:
-        raise ValueError(f"No Comptroller found for {protocol}")
-    token_mapping = {}
-    comptroller_abi = get_raw_abi("Comptroller", Protocol.compound_v2)
-    comptroller_instance = w3.eth.contract(address=address, abi=comptroller_abi)
-    markets = comptroller_instance.functions.getAllMarkets().call()
-    token_abi = get_raw_abi("CToken", Protocol.compound_v2)
-    for token in markets:
-        # make an exception for cETH (as it has no .underlying())
-        if token != c_ether:
-            token_instance = w3.eth.contract(address=token, abi=token_abi)
-            underlying_token = token_instance.functions.underlying().call()
-            token_mapping[
-                token.lower()
-            ] = underlying_token.lower()  # make k:v lowercase for consistancy
-    return token_mapping
-
 
 def get_compound_liquidations(
     traces: List[ClassifiedTrace],
-    collateral_by_c_token_address: Dict[str, str],
-    collateral_by_cr_token_address: Dict[str, str],
 ) -> List[Liquidation]:
 
     """Inspect list of classified traces and identify liquidation"""
@@ -67,23 +37,13 @@ def get_compound_liquidations(
                 trace.transaction_hash, trace.trace_address, traces
             )
             seize_trace = _get_seize_call(child_traces)
-            underlying_markets = {}
-            if trace.protocol == Protocol.compound_v2:
-                underlying_markets = collateral_by_c_token_address
-            elif trace.protocol == Protocol.cream:
-                underlying_markets = collateral_by_cr_token_address
 
-            if (
+            if seize_trace is not None and seize_trace.inputs is not None:
-                seize_trace is not None
-                and seize_trace.inputs is not None
-                and len(underlying_markets) != 0
-            ):
                 c_token_collateral = trace.inputs["cTokenCollateral"]
                 if trace.abi_name == "CEther":
                     liquidations.append(
                         Liquidation(
                             liquidated_user=trace.inputs["borrower"],
-                            collateral_token_address=ETH_TOKEN_ADDRESS,  # WETH since all cEther liquidations provide Ether
                             debt_token_address=c_token_collateral,
                             liquidator_user=seize_trace.inputs["liquidator"],
                             debt_purchase_amount=trace.value,
@@ -97,13 +57,9 @@ def get_compound_liquidations(
                 elif (
                     trace.abi_name == "CToken"
                 ):  # cToken liquidations where liquidator pays back via token transfer
-                    c_token_address = trace.to_address
                     liquidations.append(
                         Liquidation(
                             liquidated_user=trace.inputs["borrower"],
-                            collateral_token_address=underlying_markets[
-                                c_token_address
-                            ],
                             debt_token_address=c_token_collateral,
                             liquidator_user=seize_trace.inputs["liquidator"],
                             debt_purchase_amount=trace.inputs["repayAmount"],

mev_inspect/liquidations.py

@@ -1,6 +1,7 @@
 from typing import List
 
 from mev_inspect.aave_liquidations import get_aave_liquidations
+from mev_inspect.compound_liquidations import get_compound_liquidations
 from mev_inspect.schemas.traces import (
     ClassifiedTrace,
     Classification,
@@ -20,4 +21,5 @@ def get_liquidations(
     classified_traces: List[ClassifiedTrace],
 ) -> List[Liquidation]:
     aave_liquidations = get_aave_liquidations(classified_traces)
-    return aave_liquidations
+    comp_liquidations = get_compound_liquidations(classified_traces)
+    return aave_liquidations + comp_liquidations

mev_inspect/models/liquidations.py

@@ -8,7 +8,6 @@ class LiquidationModel(Base):
 
     liquidated_user = Column(String, nullable=False)
     liquidator_user = Column(String, nullable=False)
-    collateral_token_address = Column(String, nullable=False)
     debt_token_address = Column(String, nullable=False)
     debt_purchase_amount = Column(Numeric, nullable=False)
     received_amount = Column(Numeric, nullable=False)

mev_inspect/schemas/liquidations.py

@@ -6,7 +6,6 @@ from mev_inspect.schemas.traces import Protocol
 class Liquidation(BaseModel):
     liquidated_user: str
     liquidator_user: str
-    collateral_token_address: str
     debt_token_address: str
     debt_purchase_amount: int
     received_amount: int

tests/liquidation_test.py

@@ -19,7 +19,6 @@ def test_single_weth_liquidation():
         Liquidation(
             liquidated_user="0xd16404ca0a74a15e66d8ad7c925592fb02422ffe",
             liquidator_user="0x19256c009781bc2d1545db745af6dfd30c7e9cfa",
-            collateral_token_address="0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2",
             debt_token_address="0xdac17f958d2ee523a2206206994597c13d831ec7",
             debt_purchase_amount=26503300291,
             received_amount=8182733924513576561,
@@ -50,7 +49,6 @@ def test_single_liquidation():
         Liquidation(
             liquidated_user="0x8d8d912fe4db5917da92d14fea05225b803c359c",
             liquidator_user="0xf2d9e54f0e317b8ac94825b2543908e7552fe9c7",
-            collateral_token_address="0x80fb784b7ed66730e8b1dbd9820afd29931aab03",
             debt_token_address="0xdac17f958d2ee523a2206206994597c13d831ec7",
             debt_purchase_amount=1069206535,
             received_amount=2657946947610159065393,
@@ -81,7 +79,6 @@ def test_single_liquidation_with_atoken_payback():
         Liquidation(
             liquidated_user="0x3d2b6eacd1bca51af57ed8b3ff9ef0bd8ee8c56d",
             liquidator_user="0x887668f2dc9612280243f2a6ef834cecf456654e",
-            collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
             debt_token_address="0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2",
             debt_purchase_amount=767615458043667978,
             received_amount=113993647930952952550,
@@ -111,7 +108,6 @@ def test_multiple_liquidations_in_block():
     liquidation1 = Liquidation(
         liquidated_user="0x6c6541ae8a7c6a6f968124a5ff2feac8f0c7875b",
         liquidator_user="0x7185e240d8e9e2d692cbc68d30eecf965e9a7feb",
-        collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
         debt_token_address="0x4fabb145d64652a948d72533023f6e7a623c7c53",
         debt_purchase_amount=457700000000000000000,
         received_amount=10111753901939162887,
@@ -125,7 +121,6 @@ def test_multiple_liquidations_in_block():
     liquidation2 = Liquidation(
         liquidated_user="0x6c6541ae8a7c6a6f968124a5ff2feac8f0c7875b",
         liquidator_user="0x7185e240d8e9e2d692cbc68d30eecf965e9a7feb",
-        collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
         debt_token_address="0x0000000000085d4780b73119b644ae5ecd22b376",
         debt_purchase_amount=497030000000000000000,
         received_amount=21996356316098208090,
@@ -139,7 +134,6 @@ def test_multiple_liquidations_in_block():
     liquidation3 = Liquidation(
         liquidated_user="0xda874f844389df33c0fad140df4970fe1b366726",
         liquidator_user="0x7185e240d8e9e2d692cbc68d30eecf965e9a7feb",
-        collateral_token_address="0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2",
         debt_token_address="0x57ab1ec28d129707052df4df418d58a2d46d5f51",
         debt_purchase_amount=447810000000000000000,
         received_amount=121531358145247546,
@@ -169,7 +163,6 @@ def test_liquidations_with_eth_transfer():
     liquidation1 = Liquidation(
         liquidated_user="0xad346c7762f74c78da86d2941c6eb546e316fbd0",
         liquidator_user="0x27239549dd40e1d60f5b80b0c4196923745b1fd2",
-        collateral_token_address=ETH_TOKEN_ADDRESS,
         debt_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
         debt_purchase_amount=1809152000000000000,
         received_amount=15636807387264000,
@@ -183,7 +176,6 @@ def test_liquidations_with_eth_transfer():
     liquidation2 = Liquidation(
         liquidated_user="0xad346c7762f74c78da86d2941c6eb546e316fbd0",
         liquidator_user="0x27239549dd40e1d60f5b80b0c4196923745b1fd2",
-        collateral_token_address=ETH_TOKEN_ADDRESS,
         debt_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
         debt_purchase_amount=1809152000000000000,
         received_amount=8995273139160873,

tests/test_compound.py

@@ -2,7 +2,6 @@ from mev_inspect.compound_liquidations import get_compound_liquidations
 from mev_inspect.schemas.liquidations import Liquidation
 from mev_inspect.schemas.traces import Protocol
 from mev_inspect.classifiers.trace import TraceClassifier
-from mev_inspect.transfers import ETH_TOKEN_ADDRESS
 from tests.utils import load_test_block, load_comp_markets, load_cream_markets
 
 comp_markets = load_comp_markets()
@@ -19,7 +18,6 @@ def test_c_ether_liquidations():
         Liquidation(
             liquidated_user="0xb5535a3681cf8d5431b8acfd779e2f79677ecce9",
             liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
-            collateral_token_address=ETH_TOKEN_ADDRESS,
             debt_token_address="0x39aa39c021dfbae8fac545936693ac917d5e7563",
             debt_purchase_amount=268066492249420078,
             received_amount=4747650169097,
@@ -32,7 +30,7 @@ def test_c_ether_liquidations():
     block = load_test_block(block_number)
     trace_classifier = TraceClassifier()
     classified_traces = trace_classifier.classify(block.traces)
-    result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
+    result = get_compound_liquidations(classified_traces)
     assert result == liquidations
 
     block_number = 13207907
@@ -44,7 +42,6 @@ def test_c_ether_liquidations():
         Liquidation(
             liquidated_user="0x45df6f00166c3fb77dc16b9e47ff57bc6694e898",
             liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
-            collateral_token_address=ETH_TOKEN_ADDRESS,
             debt_token_address="0x35a18000230da775cac24873d00ff85bccded550",
             debt_purchase_amount=414547860568297082,
             received_amount=321973320649,
@@ -58,7 +55,7 @@ def test_c_ether_liquidations():
     block = load_test_block(block_number)
     trace_classifier = TraceClassifier()
     classified_traces = trace_classifier.classify(block.traces)
-    result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
+    result = get_compound_liquidations(classified_traces)
     assert result == liquidations
 
     block_number = 13298725
@@ -70,7 +67,6 @@ def test_c_ether_liquidations():
         Liquidation(
             liquidated_user="0xacbcf5d2970eef25f02a27e9d9cd31027b058b9b",
             liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
-            collateral_token_address=ETH_TOKEN_ADDRESS,
             debt_token_address="0x35a18000230da775cac24873d00ff85bccded550",
             debt_purchase_amount=1106497772527562662,
             received_amount=910895850496,
@@ -83,7 +79,7 @@ def test_c_ether_liquidations():
     block = load_test_block(block_number)
     trace_classifier = TraceClassifier()
     classified_traces = trace_classifier.classify(block.traces)
-    result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
+    result = get_compound_liquidations(classified_traces)
     assert result == liquidations
 
 
@@ -97,7 +93,6 @@ def test_c_token_liquidation():
         Liquidation(
             liquidated_user="0xacdd5528c1c92b57045041b5278efa06cdade4d8",
             liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
-            collateral_token_address="0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48",
             debt_token_address="0x70e36f6bf80a52b3b46b3af8e106cc0ed743e8e4",
             debt_purchase_amount=1207055531,
             received_amount=21459623305,
@@ -110,7 +105,7 @@ def test_c_token_liquidation():
     block = load_test_block(block_number)
     trace_classifier = TraceClassifier()
     classified_traces = trace_classifier.classify(block.traces)
-    result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
+    result = get_compound_liquidations(classified_traces)
     assert result == liquidations
 
 
@@ -124,7 +119,6 @@ def test_cream_token_liquidation():
         Liquidation(
             liquidated_user="0x46bf9479dc569bc796b7050344845f6564d45fba",
             liquidator_user="0xa2863cad9c318669660eb4eca8b3154b90fb4357",
-            collateral_token_address="0x514910771af9ca656af840dff83e8264ecf986ca",
             debt_token_address="0x44fbebd2f576670a6c33f6fc0b00aa8c5753b322",
             debt_purchase_amount=14857434973806369550,
             received_amount=1547215810826,
@@ -137,5 +131,5 @@ def test_cream_token_liquidation():
     block = load_test_block(block_number)
     trace_classifier = TraceClassifier()
     classified_traces = trace_classifier.classify(block.traces)
-    result = get_compound_liquidations(classified_traces, comp_markets, cream_markets)
+    result = get_compound_liquidations(classified_traces)
     assert result == liquidations