Initial JIT Liquidity Classifier Commit

alembic/versions/ceb5976b37dd_add_swap_jit_liquidity_join_table.py

@@ -0,0 +1,38 @@
+"""add_swap_jit_liquidity_join_table
+
+Revision ID: ceb5976b37dd
+Revises: 5c5375de15fd
+Create Date: 2022-04-19 18:34:26.332094
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'ceb5976b37dd'
+down_revision = '5c5375de15fd'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    sa.create_table(
+        "jit_liquidity_swaps",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("jit_liquidity_id", sa.String(1024), primary_key=True),
+        sa.Column("swap_transaction_hash", sa.String(66), primary_key=True),
+        sa.Column("swap_trace_address", sa.ARRAY(sa.Integer), primary_key=True),
+        sa.ForeignKeyConstraint(
+            ["jit_liquidity_id"], ["jit_liquidity.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["swap_transaction_hash", "swap_trace_address"],
+            ["swaps.transaction_hash", "swaps.trace_address"],
+            ondelete="CASCADE",
+        )
+    )
+
+
+def downgrade():
+    op.drop_table("jit_liquidity_swaps")

mev_inspect/classifiers/specs/uniswap.py

@@ -1,9 +1,9 @@
 from typing import List, Optional
 
 from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
-from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier
+from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier, Classifier
 from mev_inspect.schemas.swaps import Swap
-from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol, Classification
 from mev_inspect.schemas.transfers import Transfer
 
 UNISWAP_V2_PAIR_ABI_NAME = "UniswapV2Pair"
@@ -42,6 +42,18 @@ class UniswapV2SwapClassifier(SwapClassifier):
         return swap
 
 
+class LiquidityMintClassifier(Classifier):
+    @staticmethod
+    def get_classification() -> Classification:
+        return Classification.liquidity_mint
+
+
+class LiquidityBurnClassifier(Classifier):
+    @staticmethod
+    def get_classification() -> Classification:
+        return Classification.liquidity_burn
+
+
 UNISWAP_V3_CONTRACT_SPECS = [
     ClassifierSpec(
         abi_name="UniswapV3Factory",
@@ -106,6 +118,8 @@ UNISWAP_V3_GENERAL_SPECS = [
         protocol=Protocol.uniswap_v3,
         classifiers={
             "swap(address,bool,int256,uint160,bytes)": UniswapV3SwapClassifier,
+            "mint(address,int24,int24,uint128,bytes)": LiquidityMintClassifier,
+            "burn(int24,int24,uint128)": LiquidityBurnClassifier,
         },
     ),
     ClassifierSpec(

mev_inspect/crud/jit_liquidity.py

@@ -0,0 +1,70 @@
+from typing import List
+from uuid import uuid4
+
+from mev_inspect.schemas.jit_liquidity import JITLiquidity
+from mev_inspect.models.jit_liquidity import JITLiquidityModel
+
+from .shared import delete_by_block_range
+
+
+def delete_jit_liquidity_for_blocks(
+        db_session,
+        after_block_number: int,
+        before_block_number: int,
+) -> None:
+    delete_by_block_range(
+        db_session,
+        JITLiquidityModel,
+        after_block_number,
+        before_block_number,
+    )
+    db_session.commit()
+
+
+def write_jit_liquidity(
+        db_session,
+        jit_liquidity_instances: List[JITLiquidity]
+) -> None:
+    jit_liquidity_models = []
+    swap_jit_liquidity_ids = []
+    for jit_liquidity in jit_liquidity_instances:
+        jit_liquidity_id = str(uuid4())
+        jit_liquidity_models.append(JITLiquidityModel(
+            id=jit_liquidity_id,
+            block_number=jit_liquidity.block_number,
+            bot_address=jit_liquidity.bot_address,
+            pool_address=jit_liquidity.pool_address,
+            token0_address=jit_liquidity.token0_address,
+            token1_address=jit_liquidity.token1_address,
+            mint_transaction_hash=jit_liquidity.mint_transaction_hash,
+            mint_transaction_trace=jit_liquidity.mint_trace,
+            burn_transaction_hash=jit_liquidity.burn_transaction_hash,
+            burn_transaction_trace=jit_liquidity.burn_trace,
+            mint_token0_amount=jit_liquidity.mint_token0_amount,
+            mint_token1_amount=jit_liquidity.mint_token1_amount,
+            burn_token0_amoun=jit_liquidity.burn_token0_amount,
+            burn_token1_amount=jit_liquidity.burn_token1_amount,
+            token0_swap_volume=jit_liquidity.token0_swap_volume,
+            token1_swap_volume=jit_liquidity.token1_swap_volume
+        ))
+
+        for swap in jit_liquidity.swaps:
+            swap_jit_liquidity_ids.append({
+                "jit_liquidity_id": jit_liquidity_id,
+                "swap_transaction_hash": swap.transaction_hash,
+                "swap_trace_address": swap.trace_address
+            })
+
+    if len(jit_liquidity_models) > 0:
+        db_session.bulk_save_objects(jit_liquidity_models)
+        db_session.execute(
+            """
+            INSERT INTO jit_liquidity_swaps
+            (jit_liquidity_id, swap_transaction_hash, swap_trace_address)
+            VALUES
+            (:jit_liquidity_id, :swap_transaction_hash, :swap_trace_address)
+            """,
+            params=swap_jit_liquidity_ids
+        )
+
+        db_session.commit()

mev_inspect/inspect_block.py

@@ -53,6 +53,9 @@ from mev_inspect.schemas.traces import ClassifiedTrace
 from mev_inspect.schemas.transfers import Transfer
 from mev_inspect.swaps import get_swaps
 from mev_inspect.transfers import get_transfers
+from mev_inspect.jit_liquidity import get_jit_liquidity
+from mev_inspect.schemas.jit_liquidity import JITLiquidity
+from mev_inspect.crud.jit_liquidity import delete_jit_liquidity_for_blocks, write_jit_liquidity
 
 logger = logging.getLogger(__name__)
 
@@ -100,6 +103,7 @@ async def inspect_many_blocks(
     all_miner_payments: List[MinerPayment] = []
 
     all_nft_trades: List[NftTrade] = []
+    all_jit_liquidity: List[JITLiquidity] = []
 
     for block_number in range(after_block_number, before_block_number):
         block = await create_from_block_number(
@@ -149,6 +153,9 @@ async def inspect_many_blocks(
         nft_trades = get_nft_trades(classified_traces)
         logger.info(f"Block: {block_number} -- Found {len(nft_trades)} nft trades")
 
+        jit_liquidity = get_jit_liquidity(classified_traces, swaps)
+        logger.info(f"Block: {block_number} -- Found {len(jit_liquidity)} jit liquidity instances")
+
         miner_payments = get_miner_payments(
             block.miner, block.base_fee_per_gas, classified_traces, block.receipts
         )
@@ -167,6 +174,8 @@ async def inspect_many_blocks(
 
         all_nft_trades.extend(nft_trades)
 
+        all_jit_liquidity.extend(jit_liquidity)
+
         all_miner_payments.extend(miner_payments)
 
     logger.info("Writing data")
@@ -222,6 +231,11 @@ async def inspect_many_blocks(
     )
     write_nft_trades(inspect_db_session, all_nft_trades)
 
+    delete_jit_liquidity_for_blocks(
+        inspect_db_session, after_block_number, before_block_number
+    )
+    write_jit_liquidity(inspect_db_session, all_jit_liquidity)
+
     delete_miner_payments_for_blocks(
         inspect_db_session, after_block_number, before_block_number
     )

mev_inspect/jit_liquidity.py

@@ -0,0 +1,121 @@
+from typing import List
+
+from mev_inspect.schemas.jit_liquidity import JITLiquidity
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.transfers import Transfer
+from mev_inspect.transfers import get_net_transfers
+from mev_inspect.traces import is_child_trace_address
+from mev_inspect.schemas.traces import ClassifiedTrace, DecodedCallTrace, Classification, Protocol
+
+LIQUIDITY_MINT_ROUTERS = [
+    "0xC36442b4a4522E871399CD717aBDD847Ab11FE88".lower(),  # Uniswap V3 NFT Position Manager
+]
+
+
+def get_jit_liquidity(
+        classified_traces: List[ClassifiedTrace],
+        swaps: List[Swap]
+) -> List[JITLiquidity]:
+    jit_liquidity_instances: List[JITLiquidity] = []
+
+    for index, trace in enumerate(classified_traces):
+
+        if not isinstance(trace, DecodedCallTrace):
+            continue
+
+        if trace.classification == Classification.liquidity_mint and trace.protocol == Protocol.uniswap_v3:
+            i = index + 1
+            while i < len(classified_traces):
+                forward_search_trace = classified_traces[i]
+                if forward_search_trace.classification == Classification.liquidity_burn:
+                    if forward_search_trace.to_address == trace.to_address:
+                        jit_liquidity_instances.append(
+                            _parse_jit_liquidity_instance(trace, forward_search_trace, classified_traces, swaps)
+                        )
+                i += 1
+
+    return jit_liquidity_instances
+
+
+def _parse_jit_liquidity_instance(
+        mint_trace: ClassifiedTrace,
+        burn_trace: ClassifiedTrace,
+        classified_traces: List[ClassifiedTrace],
+        swaps: List[Swap],
+) -> JITLiquidity:
+    valid_swaps = list(filter(
+        lambda t: mint_trace.transaction_position < t.transaction_position < burn_trace.transaction_position,
+        swaps
+    ))
+    net_transfers = get_net_transfers(list(filter(
+        lambda t: t.transaction_hash in [mint_trace.transaction_hash, burn_trace.transaction_hash],
+        classified_traces)))
+
+    jit_swaps: List[Swap] = []
+    token0_swap_volume, token1_swap_volume = 0, 0
+
+    mint_transfers: List[Transfer] = list(filter(
+        lambda t: t.transaction_hash == mint_trace.transaction_hash and t.to_address == mint_trace.to_address,
+        net_transfers))
+    burn_transfers: List[Transfer] = list(filter(
+        lambda t: t.transaction_hash == burn_trace.transaction_hash and t.from_address == burn_trace.to_address,
+        net_transfers))
+
+    if len(mint_transfers) == 2 and len(burn_transfers) == 2:
+        token0_address, token1_address = _get_token_order(mint_transfers[0].token_address,
+                                                          mint_transfers[1].token_address)
+    else:
+        # This is a failing/skipping case, super weird
+        return None
+
+    bot_address = _get_bot_address(mint_trace, classified_traces)
+    for swap in valid_swaps:
+        if swap.contract_address == mint_trace.to_address:
+            jit_swaps.append(swap)
+            token0_swap_volume += swap.token_in_amount if swap.token_in_address == token0_address else 0
+            token1_swap_volume += 0 if swap.token_in_address == token0_address else swap.token_in_amount
+
+    return JITLiquidity(
+        block_number=mint_trace.block_number,
+        bot_address=bot_address,
+        pool_address=mint_trace.to_address,
+        mint_transaction_hash=mint_trace.transaction_hash,
+        mint_trace=mint_trace.trace_address,
+        burn_transaction_hash=burn_trace.transaction_hash,
+        burn_trace=burn_trace.trace_address,
+        swaps=jit_swaps,
+        token0_address=token0_address,
+        token1_address=token1_address,
+        mint_token0_amount=mint_transfers[0].amount if mint_transfers[0].token_address == token0_address else mint_transfers[1].amount,
+        mint_token1_amount=mint_transfers[1].amount if mint_transfers[0].token_address == token0_address else mint_transfers[0].amount,
+        burn_token0_amount=burn_transfers[0].amount if burn_transfers[0].token_address == token0_address else burn_transfers[1].amount,
+        burn_token1_amount=burn_transfers[1].amount if burn_transfers[0].token_address == token0_address else burn_transfers[0].amount,
+        token0_swap_volume=token0_swap_volume,
+        token1_swap_volume=token1_swap_volume,
+    )
+
+
+def _get_token_order(token_a: str, token_b: str) -> (int, int):
+    token_order = True if int(token_a, 16) < int(token_b, 16) else False
+    return (token_a, token_b) if token_order else (token_b, token_a)
+
+
+def _get_bot_address(
+        mint_trace: ClassifiedTrace,
+        classified_traces: List[ClassifiedTrace]
+) -> str:
+    if mint_trace.from_address not in LIQUIDITY_MINT_ROUTERS:
+        return mint_trace.from_address
+
+    bot_trace = list(filter(
+        lambda t: t.to_address == mint_trace.from_address and t.transaction_hash == mint_trace.transaction_hash,
+        classified_traces
+    ))
+    if len(bot_trace) > 1:
+        if is_child_trace_address(bot_trace[1].trace_address, bot_trace[0].trace_address):
+            return _get_bot_address(bot_trace[0], classified_traces)
+        else:
+            return "0x" + ("0" * 40)  # get rid of this case by properly searching the trace_address
+    _get_bot_address(bot_trace[0], classified_traces)
+
+

mev_inspect/models/jit_liquidity.py

@@ -0,0 +1,25 @@
+from sqlalchemy import ARRAY, Column, Integer, Numeric, String
+
+from .base import Base
+
+
+class JITLiquidityModel(Base):
+    id = Column(String, primary_key=True)
+    block_number = Column(Numeric(), nullable=False)
+    bot_address = Column(String(42), nullable=True)
+    pool_address = Column(String(42), nullable=False)
+    token0_address = Column(String(42), nullable=True)
+    token1_address = Column(String(42), nullable=True)
+    mint_transaction_hash = Column(String(66), nullable=False)
+    mint_transaction_trace = Column(ARRAY(Integer))
+    burn_transaction_hash = Column(String(66), nullable=False)
+    burn_transaction_trace = Column(ARRAY(Integer))
+    mint_token0_amount = Column(Numeric)
+    mint_token1_amount = Column(Numeric)
+    burn_token0_amount = Column(Numeric)
+    burn_token1_amount = Column(Numeric)
+    token0_swap_volume = Column(Numeric)
+    token1_swap_volume = Column(Numeric)
+
+
+

mev_inspect/schemas/jit_liquidity.py

@@ -0,0 +1,28 @@
+from typing import List
+
+from pydantic import BaseModel
+
+
+from .swaps import Swap
+
+
+class JITLiquidity(BaseModel):
+    block_number: int
+    bot_address: str
+    pool_address: str
+    mint_transaction_hash: str
+    mint_trace: List[int]
+    burn_transaction_hash: str
+    burn_trace: List[int]
+    swaps: List[Swap]
+    token0_address: str
+    token1_address: str
+    mint_token0_amount: int
+    mint_token1_amount: int
+    burn_token0_amount: int
+    burn_token1_amount: int
+    token0_swap_volume: int
+    token1_swap_volume: int
+
+
+

mev_inspect/schemas/traces.py

@@ -34,6 +34,8 @@ class Classification(Enum):
     punk_bid = "punk_bid"
     punk_accept_bid = "punk_accept_bid"
     nft_trade = "nft_trade"
+    liquidity_mint = "liquidity_mint"
+    liquidity_burn = "liquidity_burn"
 
 
 class Protocol(Enum):

mev_inspect/transfers.py

@@ -3,7 +3,7 @@ from typing import Dict, List, Optional, Sequence
 from mev_inspect.classifiers.specs import get_classifier
 from mev_inspect.schemas.classifiers import TransferClassifier
 from mev_inspect.schemas.prices import ETH_TOKEN_ADDRESS
-from mev_inspect.schemas.traces import ClassifiedTrace, DecodedCallTrace
+from mev_inspect.schemas.traces import ClassifiedTrace, DecodedCallTrace, Classification
 from mev_inspect.schemas.transfers import Transfer
 from mev_inspect.traces import get_child_traces, is_child_trace_address
 
@@ -126,3 +126,76 @@ def remove_child_transfers_of_transfers(
         ] = existing_addresses + [transfer.trace_address]
 
     return updated_transfers
+
+
+def get_net_transfers(
+        classified_traces: List[ClassifiedTrace],
+) -> List[Transfer]:
+    """
+    Super Jank...
+    Returns the net transfers per transaction from a list of Classified Traces.
+    Ex.  if a bot transfers 200 WETH to a contract, and the contract transfers the excess WETH back to the bot,
+    the following transfer would be returned  (from_address=bot, to_address=contract, amount=150)
+    if the contract transferred 300 WETH back to the bot, the following would be returned
+    (from_address=contract, to_address=bot, amount=100). if the contract transferred back 200 WETH,
+    no transfer would be returned.
+    Additionally, ignores transfers forwarded from proxy contracts & uses initial proxy address
+    @param classified_traces:
+    @return: List of Transfer objects representing the net movement from A to B
+    """
+    found_transfers: List[list] = []
+    return_transfers: List[Transfer] = []
+    for trace in classified_traces:
+        if not isinstance(trace, DecodedCallTrace):
+            continue
+
+        if trace.classification == Classification.transfer:
+            if trace.from_address in [t.token_address for t in return_transfers]:  # Proxy Case
+                continue
+
+            if trace.function_signature == "transfer(address,uint256)":
+                net_search_info = [trace.inputs["recipient"], trace.to_address, trace.from_address]
+
+            else:  # trace.function_signature == "transferFrom(address,address,uint256)"
+                net_search_info = [trace.inputs["recipient"], trace.to_address, trace.inputs["sender"]]
+
+            if sorted(net_search_info) in found_transfers:
+                for index, transfer in enumerate(return_transfers):
+                    if transfer.token_address != net_search_info[1] or transfer.transaction_hash != trace.transaction_hash:
+                        continue
+
+                    if transfer.from_address == net_search_info[2] and transfer.to_address == net_search_info[0]:
+                        return_transfers[index].amount += trace.inputs["amount"]
+                        return_transfers[index].trace_address = [-1]
+                    if transfer.from_address == net_search_info[0] and transfer.to_address == net_search_info[2]:
+                        return_transfers[index].amount -= trace.inputs["amount"]
+                        return_transfers[index].trace_address = [-1]
+
+            else:
+                return_transfers.append(Transfer(
+                    block_number=trace.block_number,
+                    transaction_hash=trace.transaction_hash,
+                    trace_address=trace.trace_address,
+                    from_address=net_search_info[2],  # Janky... improve
+                    to_address=net_search_info[0],
+                    amount=trace.inputs["amount"],
+                    token_address=net_search_info[1]
+                ))
+                found_transfers.append(sorted(net_search_info))
+
+    i = 0
+    while True:
+        try:
+            transfer = return_transfers[i]
+        except IndexError:
+            break
+        if transfer.amount < 0:
+            return_transfers[i].from_address = transfer.to_address
+            return_transfers[i].to_address = transfer.from_address
+            return_transfers[i].amount = transfer.amount * -1
+        if transfer.amount == 0:
+            return_transfers.pop(i)
+            i -= 1
+        i += 1
+
+    return return_transfers

tests/test_jit_liquidity.py

@@ -0,0 +1,77 @@
+from mev_inspect.schemas.jit_liquidity import JITLiquidity
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import Protocol
+from mev_inspect.swaps import get_swaps
+from mev_inspect.jit_liquidity import get_jit_liquidity
+
+from mev_inspect.classifiers.trace import TraceClassifier
+
+from .utils import load_test_block
+
+
+def test_single_sandwich_jit_liquidity(trace_classifier: TraceClassifier):
+    print("\n")
+    test_block = load_test_block(13601096)
+
+    classified_traces = trace_classifier.classify(test_block.traces)
+    swaps = get_swaps(classified_traces)
+    jit_liquidity_instances = get_jit_liquidity(classified_traces, swaps)
+
+    # Assert Section
+
+    jit_swap = Swap(  # Double check these values
+        abi_name="UniswapV3Pool",
+        transaction_hash="0x943131400defa5db902b1df4ab5108b58527e525da3d507bd6e6465d88fa079c".lower(),
+        transaction_position=1,
+        block_number=13601096,
+        trace_address=[7, 0, 12, 1, 0],
+        contract_address="0x8ad599c3A0ff1De082011EFDDc58f1908eb6e6D8".lower(),
+        from_address="0xE592427A0AEce92De3Edee1F18E0157C05861564".lower(),
+        to_address="0xAa6E8127831c9DE45ae56bB1b0d4D4Da6e5665BD".lower(),
+        token_in_address="0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48".lower(),  # USDC Contract
+        token_in_amount=1896817745609,
+        token_out_address="0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2".lower(),
+        token_out_amount=408818202022592862626,
+        protocol=Protocol.uniswap_v3
+    )
+    expected_jit_liquidity = [
+        JITLiquidity(
+            block_number=13601096,
+            bot_address="0xa57Bd00134B2850B2a1c55860c9e9ea100fDd6CF".lower(),
+            pool_address="0x8ad599c3A0ff1De082011EFDDc58f1908eb6e6D8".lower(),
+            mint_transaction_hash="0x80e4abcb0b701e9d2c0d0fd216ef22eca5fc13904e8c7b3967bcad997480d638".lower(),
+            mint_trace=[0, 9, 1],
+            burn_transaction_hash="0x12b3d1f0e29d9093d8f3c7cce2da95edbef01aaab3794237f263da85c37c7d27".lower(),
+            burn_trace=[0, 1, 0],
+            swaps=[jit_swap],
+            token0_address="0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48",
+            token1_address="0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2",
+            mint_token0_amount=10864608891029,
+            mint_token1_amount=8281712219747858010668,
+            burn_token0_amount=12634177387879,
+            burn_token1_amount=7900319851971188832064,
+            token0_swap_volume=1896817745609,
+            token1_swap_volume=0,
+        )
+    ]
+
+    # Might be super janky but this could be done with assert jit_liquidity_instances == expected_jit_liquidity
+    assert len(jit_liquidity_instances) == 1
+    assert len(jit_liquidity_instances[0].swaps) == 1
+    assert jit_liquidity_instances[0].burn_transaction_hash == expected_jit_liquidity[0].burn_transaction_hash
+    assert jit_liquidity_instances[0].mint_transaction_hash == expected_jit_liquidity[0].mint_transaction_hash
+    assert jit_liquidity_instances[0].burn_token0_amount == expected_jit_liquidity[0].burn_token0_amount
+    assert jit_liquidity_instances[0].burn_token1_amount == expected_jit_liquidity[0].burn_token1_amount
+    assert jit_liquidity_instances[0].mint_token0_amount == expected_jit_liquidity[0].mint_token0_amount
+    assert jit_liquidity_instances[0].mint_token1_amount == expected_jit_liquidity[0].mint_token1_amount
+    assert jit_liquidity_instances[0].bot_address == expected_jit_liquidity[0].bot_address
+    assert jit_liquidity_instances[0].token0_swap_volume == expected_jit_liquidity[0].token0_swap_volume
+    assert jit_liquidity_instances[0].token1_swap_volume == expected_jit_liquidity[0].token1_swap_volume
+
+    # Swap Checks
+    assert jit_liquidity_instances[0].swaps[0].transaction_hash == jit_swap.transaction_hash
+    assert jit_liquidity_instances[0].swaps[0].trace_address == jit_swap.trace_address
+
+
+
+