From 5abfc38e121e0c06aa30574fce165d1f98a22a72 Mon Sep 17 00:00:00 2001 From: elicb Date: Tue, 19 Apr 2022 20:00:44 -0700 Subject: [PATCH] Actually reading the docs and installing the pre-commit --- ...b37dd_add_swap_jit_liquidity_join_table.py | 9 +- mev_inspect/classifiers/specs/uniswap.py | 4 +- mev_inspect/crud/jit_liquidity.py | 63 +++++----- mev_inspect/inspect_block.py | 13 +- mev_inspect/jit_liquidity.py | 118 ++++++++++++------ mev_inspect/models/jit_liquidity.py | 3 - mev_inspect/schemas/jit_liquidity.py | 8 +- mev_inspect/transfers.py | 55 +++++--- tests/test_jit_liquidity.py | 60 ++++++--- 9 files changed, 210 insertions(+), 123 deletions(-) diff --git a/alembic/versions/ceb5976b37dd_add_swap_jit_liquidity_join_table.py b/alembic/versions/ceb5976b37dd_add_swap_jit_liquidity_join_table.py index 0df4ca9..a9f8c02 100644 --- a/alembic/versions/ceb5976b37dd_add_swap_jit_liquidity_join_table.py +++ b/alembic/versions/ceb5976b37dd_add_swap_jit_liquidity_join_table.py @@ -5,13 +5,12 @@ Revises: 5c5375de15fd Create Date: 2022-04-19 18:34:26.332094 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. -revision = 'ceb5976b37dd' -down_revision = '5c5375de15fd' +revision = "ceb5976b37dd" +down_revision = "5c5375de15fd" branch_labels = None depends_on = None @@ -30,7 +29,7 @@ def upgrade(): ["swap_transaction_hash", "swap_trace_address"], ["swaps.transaction_hash", "swaps.trace_address"], ondelete="CASCADE", - ) + ), ) diff --git a/mev_inspect/classifiers/specs/uniswap.py b/mev_inspect/classifiers/specs/uniswap.py index b0a0a2a..26e0943 100644 --- a/mev_inspect/classifiers/specs/uniswap.py +++ b/mev_inspect/classifiers/specs/uniswap.py @@ -1,9 +1,9 @@ from typing import List, Optional from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers -from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier, Classifier +from mev_inspect.schemas.classifiers import Classifier, ClassifierSpec, SwapClassifier from mev_inspect.schemas.swaps import Swap -from mev_inspect.schemas.traces import DecodedCallTrace, Protocol, Classification +from mev_inspect.schemas.traces import Classification, DecodedCallTrace, Protocol from mev_inspect.schemas.transfers import Transfer UNISWAP_V2_PAIR_ABI_NAME = "UniswapV2Pair" diff --git a/mev_inspect/crud/jit_liquidity.py b/mev_inspect/crud/jit_liquidity.py index 5411a8a..b0d01b1 100644 --- a/mev_inspect/crud/jit_liquidity.py +++ b/mev_inspect/crud/jit_liquidity.py @@ -1,16 +1,16 @@ from typing import List from uuid import uuid4 -from mev_inspect.schemas.jit_liquidity import JITLiquidity from mev_inspect.models.jit_liquidity import JITLiquidityModel +from mev_inspect.schemas.jit_liquidity import JITLiquidity from .shared import delete_by_block_range def delete_jit_liquidity_for_blocks( - db_session, - after_block_number: int, - before_block_number: int, + db_session, + after_block_number: int, + before_block_number: int, ) -> None: delete_by_block_range( db_session, @@ -22,38 +22,41 @@ def delete_jit_liquidity_for_blocks( def write_jit_liquidity( - db_session, - jit_liquidity_instances: List[JITLiquidity] + db_session, jit_liquidity_instances: List[JITLiquidity] ) -> None: jit_liquidity_models = [] swap_jit_liquidity_ids = [] for jit_liquidity in jit_liquidity_instances: jit_liquidity_id = str(uuid4()) - jit_liquidity_models.append(JITLiquidityModel( - id=jit_liquidity_id, - block_number=jit_liquidity.block_number, - bot_address=jit_liquidity.bot_address, - pool_address=jit_liquidity.pool_address, - token0_address=jit_liquidity.token0_address, - token1_address=jit_liquidity.token1_address, - mint_transaction_hash=jit_liquidity.mint_transaction_hash, - mint_transaction_trace=jit_liquidity.mint_trace, - burn_transaction_hash=jit_liquidity.burn_transaction_hash, - burn_transaction_trace=jit_liquidity.burn_trace, - mint_token0_amount=jit_liquidity.mint_token0_amount, - mint_token1_amount=jit_liquidity.mint_token1_amount, - burn_token0_amoun=jit_liquidity.burn_token0_amount, - burn_token1_amount=jit_liquidity.burn_token1_amount, - token0_swap_volume=jit_liquidity.token0_swap_volume, - token1_swap_volume=jit_liquidity.token1_swap_volume - )) + jit_liquidity_models.append( + JITLiquidityModel( + id=jit_liquidity_id, + block_number=jit_liquidity.block_number, + bot_address=jit_liquidity.bot_address, + pool_address=jit_liquidity.pool_address, + token0_address=jit_liquidity.token0_address, + token1_address=jit_liquidity.token1_address, + mint_transaction_hash=jit_liquidity.mint_transaction_hash, + mint_transaction_trace=jit_liquidity.mint_trace, + burn_transaction_hash=jit_liquidity.burn_transaction_hash, + burn_transaction_trace=jit_liquidity.burn_trace, + mint_token0_amount=jit_liquidity.mint_token0_amount, + mint_token1_amount=jit_liquidity.mint_token1_amount, + burn_token0_amoun=jit_liquidity.burn_token0_amount, + burn_token1_amount=jit_liquidity.burn_token1_amount, + token0_swap_volume=jit_liquidity.token0_swap_volume, + token1_swap_volume=jit_liquidity.token1_swap_volume, + ) + ) for swap in jit_liquidity.swaps: - swap_jit_liquidity_ids.append({ - "jit_liquidity_id": jit_liquidity_id, - "swap_transaction_hash": swap.transaction_hash, - "swap_trace_address": swap.trace_address - }) + swap_jit_liquidity_ids.append( + { + "jit_liquidity_id": jit_liquidity_id, + "swap_transaction_hash": swap.transaction_hash, + "swap_trace_address": swap.trace_address, + } + ) if len(jit_liquidity_models) > 0: db_session.bulk_save_objects(jit_liquidity_models) @@ -64,7 +67,7 @@ def write_jit_liquidity( VALUES (:jit_liquidity_id, :swap_transaction_hash, :swap_trace_address) """, - params=swap_jit_liquidity_ids + params=swap_jit_liquidity_ids, ) db_session.commit() diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py index b577b99..4ef00b8 100644 --- a/mev_inspect/inspect_block.py +++ b/mev_inspect/inspect_block.py @@ -9,6 +9,10 @@ from mev_inspect.block import create_from_block_number from mev_inspect.classifiers.trace import TraceClassifier from mev_inspect.crud.arbitrages import delete_arbitrages_for_blocks, write_arbitrages from mev_inspect.crud.blocks import delete_blocks, write_blocks +from mev_inspect.crud.jit_liquidity import ( + delete_jit_liquidity_for_blocks, + write_jit_liquidity, +) from mev_inspect.crud.liquidations import ( delete_liquidations_for_blocks, write_liquidations, @@ -34,6 +38,7 @@ from mev_inspect.crud.traces import ( write_classified_traces, ) from mev_inspect.crud.transfers import delete_transfers_for_blocks, write_transfers +from mev_inspect.jit_liquidity import get_jit_liquidity from mev_inspect.liquidations import get_liquidations from mev_inspect.miner_payments import get_miner_payments from mev_inspect.nft_trades import get_nft_trades @@ -41,6 +46,7 @@ from mev_inspect.punks import get_punk_bid_acceptances, get_punk_bids, get_punk_ from mev_inspect.sandwiches import get_sandwiches from mev_inspect.schemas.arbitrages import Arbitrage from mev_inspect.schemas.blocks import Block +from mev_inspect.schemas.jit_liquidity import JITLiquidity from mev_inspect.schemas.liquidations import Liquidation from mev_inspect.schemas.miner_payments import MinerPayment from mev_inspect.schemas.nft_trades import NftTrade @@ -53,9 +59,6 @@ from mev_inspect.schemas.traces import ClassifiedTrace from mev_inspect.schemas.transfers import Transfer from mev_inspect.swaps import get_swaps from mev_inspect.transfers import get_transfers -from mev_inspect.jit_liquidity import get_jit_liquidity -from mev_inspect.schemas.jit_liquidity import JITLiquidity -from mev_inspect.crud.jit_liquidity import delete_jit_liquidity_for_blocks, write_jit_liquidity logger = logging.getLogger(__name__) @@ -154,7 +157,9 @@ async def inspect_many_blocks( logger.info(f"Block: {block_number} -- Found {len(nft_trades)} nft trades") jit_liquidity = get_jit_liquidity(classified_traces, swaps) - logger.info(f"Block: {block_number} -- Found {len(jit_liquidity)} jit liquidity instances") + logger.info( + f"Block: {block_number} -- Found {len(jit_liquidity)} jit liquidity instances" + ) miner_payments = get_miner_payments( block.miner, block.base_fee_per_gas, classified_traces, block.receipts diff --git a/mev_inspect/jit_liquidity.py b/mev_inspect/jit_liquidity.py index dfb9009..529759a 100644 --- a/mev_inspect/jit_liquidity.py +++ b/mev_inspect/jit_liquidity.py @@ -1,11 +1,16 @@ -from typing import List, Union, Tuple, Optional +from typing import List, Tuple, Union from mev_inspect.schemas.jit_liquidity import JITLiquidity from mev_inspect.schemas.swaps import Swap +from mev_inspect.schemas.traces import ( + Classification, + ClassifiedTrace, + DecodedCallTrace, + Protocol, +) from mev_inspect.schemas.transfers import Transfer -from mev_inspect.transfers import get_net_transfers from mev_inspect.traces import is_child_trace_address -from mev_inspect.schemas.traces import ClassifiedTrace, DecodedCallTrace, Classification, Protocol +from mev_inspect.transfers import get_net_transfers LIQUIDITY_MINT_ROUTERS = [ "0xC36442b4a4522E871399CD717aBDD847Ab11FE88".lower(), # Uniswap V3 NFT Position Manager @@ -13,8 +18,7 @@ LIQUIDITY_MINT_ROUTERS = [ def get_jit_liquidity( - classified_traces: List[ClassifiedTrace], - swaps: List[Swap] + classified_traces: List[ClassifiedTrace], swaps: List[Swap] ) -> List[JITLiquidity]: jit_liquidity_instances: List[JITLiquidity] = [] @@ -23,7 +27,10 @@ def get_jit_liquidity( if not isinstance(trace, DecodedCallTrace): continue - if trace.classification == Classification.liquidity_mint and trace.protocol == Protocol.uniswap_v3: + if ( + trace.classification == Classification.liquidity_mint + and trace.protocol == Protocol.uniswap_v3 + ): i = index + 1 while i < len(classified_traces): forward_search_trace = classified_traces[i] @@ -41,32 +48,51 @@ def get_jit_liquidity( def _parse_jit_liquidity_instance( - mint_trace: ClassifiedTrace, - burn_trace: ClassifiedTrace, - classified_traces: List[ClassifiedTrace], - swaps: List[Swap], + mint_trace: ClassifiedTrace, + burn_trace: ClassifiedTrace, + classified_traces: List[ClassifiedTrace], + swaps: List[Swap], ) -> Union[JITLiquidity, None]: - valid_swaps = list(filter( - lambda t: mint_trace.transaction_position < t.transaction_position < burn_trace.transaction_position, - swaps - )) - net_transfers = get_net_transfers(list(filter( - lambda t: t.transaction_hash in [mint_trace.transaction_hash, burn_trace.transaction_hash], - classified_traces))) + valid_swaps = list( + filter( + lambda t: mint_trace.transaction_position + < t.transaction_position + < burn_trace.transaction_position, + swaps, + ) + ) + net_transfers = get_net_transfers( + list( + filter( + lambda t: t.transaction_hash + in [mint_trace.transaction_hash, burn_trace.transaction_hash], + classified_traces, + ) + ) + ) jit_swaps: List[Swap] = [] token0_swap_volume, token1_swap_volume = 0, 0 - mint_transfers: List[Transfer] = list(filter( - lambda t: t.transaction_hash == mint_trace.transaction_hash and t.to_address == mint_trace.to_address, - net_transfers)) - burn_transfers: List[Transfer] = list(filter( - lambda t: t.transaction_hash == burn_trace.transaction_hash and t.from_address == burn_trace.to_address, - net_transfers)) + mint_transfers: List[Transfer] = list( + filter( + lambda t: t.transaction_hash == mint_trace.transaction_hash + and t.to_address == mint_trace.to_address, + net_transfers, + ) + ) + burn_transfers: List[Transfer] = list( + filter( + lambda t: t.transaction_hash == burn_trace.transaction_hash + and t.from_address == burn_trace.to_address, + net_transfers, + ) + ) if len(mint_transfers) == 2 and len(burn_transfers) == 2: - token0_address, token1_address = _get_token_order(mint_transfers[0].token_address, - mint_transfers[1].token_address) + token0_address, token1_address = _get_token_order( + mint_transfers[0].token_address, mint_transfers[1].token_address + ) else: # This is a failing/skipping case, super weird return None @@ -75,8 +101,12 @@ def _parse_jit_liquidity_instance( for swap in valid_swaps: if swap.contract_address == mint_trace.to_address: jit_swaps.append(swap) - token0_swap_volume += swap.token_in_amount if swap.token_in_address == token0_address else 0 - token1_swap_volume += 0 if swap.token_in_address == token0_address else swap.token_in_amount + token0_swap_volume += ( + swap.token_in_amount if swap.token_in_address == token0_address else 0 + ) + token1_swap_volume += ( + 0 if swap.token_in_address == token0_address else swap.token_in_amount + ) token_order = mint_transfers[0].token_address == token0_address @@ -91,10 +121,18 @@ def _parse_jit_liquidity_instance( swaps=jit_swaps, token0_address=token0_address, token1_address=token1_address, - mint_token0_amount=mint_transfers[0].amount if token_order else mint_transfers[1].amount, - mint_token1_amount=mint_transfers[1].amount if token_order else mint_transfers[0].amount, - burn_token0_amount=burn_transfers[0].amount if token_order else burn_transfers[1].amount, - burn_token1_amount=burn_transfers[1].amount if token_order else burn_transfers[0].amount, + mint_token0_amount=mint_transfers[0].amount + if token_order + else mint_transfers[1].amount, + mint_token1_amount=mint_transfers[1].amount + if token_order + else mint_transfers[0].amount, + burn_token0_amount=burn_transfers[0].amount + if token_order + else burn_transfers[1].amount, + burn_token1_amount=burn_transfers[1].amount + if token_order + else burn_transfers[0].amount, token0_swap_volume=token0_swap_volume, token1_swap_volume=token1_swap_volume, ) @@ -106,17 +144,21 @@ def _get_token_order(token_a: str, token_b: str) -> Tuple[str, str]: def _get_bot_address( # Janky and a half... - mint_trace: ClassifiedTrace, - classified_traces: List[ClassifiedTrace] + mint_trace: ClassifiedTrace, classified_traces: List[ClassifiedTrace] ) -> Union[str, None]: if mint_trace.from_address in LIQUIDITY_MINT_ROUTERS: - bot_trace = list(filter( - lambda t: t.to_address == mint_trace.from_address and t.transaction_hash == mint_trace.transaction_hash, - classified_traces - )) + bot_trace = list( + filter( + lambda t: t.to_address == mint_trace.from_address + and t.transaction_hash == mint_trace.transaction_hash, + classified_traces, + ) + ) if len(bot_trace) == 1: return _get_bot_address(bot_trace[0], classified_traces) - elif is_child_trace_address(bot_trace[1].trace_address, bot_trace[0].trace_address): + elif is_child_trace_address( + bot_trace[1].trace_address, bot_trace[0].trace_address + ): return _get_bot_address(bot_trace[0], classified_traces) else: return None diff --git a/mev_inspect/models/jit_liquidity.py b/mev_inspect/models/jit_liquidity.py index ed2f796..df2c8e1 100644 --- a/mev_inspect/models/jit_liquidity.py +++ b/mev_inspect/models/jit_liquidity.py @@ -20,6 +20,3 @@ class JITLiquidityModel(Base): burn_token1_amount = Column(Numeric) token0_swap_volume = Column(Numeric) token1_swap_volume = Column(Numeric) - - - diff --git a/mev_inspect/schemas/jit_liquidity.py b/mev_inspect/schemas/jit_liquidity.py index f5aafc9..440580d 100644 --- a/mev_inspect/schemas/jit_liquidity.py +++ b/mev_inspect/schemas/jit_liquidity.py @@ -1,14 +1,13 @@ -from typing import List +from typing import List, Union from pydantic import BaseModel - from .swaps import Swap class JITLiquidity(BaseModel): block_number: int - bot_address: str + bot_address: Union[str, None] pool_address: str mint_transaction_hash: str mint_trace: List[int] @@ -23,6 +22,3 @@ class JITLiquidity(BaseModel): burn_token1_amount: int token0_swap_volume: int token1_swap_volume: int - - - diff --git a/mev_inspect/transfers.py b/mev_inspect/transfers.py index 2fa229b..dc2577b 100644 --- a/mev_inspect/transfers.py +++ b/mev_inspect/transfers.py @@ -3,7 +3,7 @@ from typing import Dict, List, Optional, Sequence from mev_inspect.classifiers.specs import get_classifier from mev_inspect.schemas.classifiers import TransferClassifier from mev_inspect.schemas.prices import ETH_TOKEN_ADDRESS -from mev_inspect.schemas.traces import ClassifiedTrace, DecodedCallTrace, Classification +from mev_inspect.schemas.traces import Classification, ClassifiedTrace, DecodedCallTrace from mev_inspect.schemas.transfers import Transfer from mev_inspect.traces import get_child_traces, is_child_trace_address @@ -129,7 +129,7 @@ def remove_child_transfers_of_transfers( def get_net_transfers( - classified_traces: List[ClassifiedTrace], + classified_traces: List[ClassifiedTrace], ) -> List[Transfer]: """ Super Jank... @@ -150,37 +150,58 @@ def get_net_transfers( continue if trace.classification == Classification.transfer: - if trace.from_address in [t.token_address for t in return_transfers]: # Proxy Case + if trace.from_address in [ + t.token_address for t in return_transfers + ]: # Proxy Case continue if trace.function_signature == "transfer(address,uint256)": - net_search_info = [trace.inputs["recipient"], trace.to_address, trace.from_address] + net_search_info = [ + trace.inputs["recipient"], + trace.to_address, + trace.from_address, + ] else: # trace.function_signature == "transferFrom(address,address,uint256)" - net_search_info = [trace.inputs["recipient"], trace.to_address, trace.inputs["sender"]] + net_search_info = [ + trace.inputs["recipient"], + trace.to_address, + trace.inputs["sender"], + ] if sorted(net_search_info) in found_transfers: for index, transfer in enumerate(return_transfers): - if transfer.token_address != net_search_info[1] or transfer.transaction_hash != trace.transaction_hash: + if ( + transfer.token_address != net_search_info[1] + or transfer.transaction_hash != trace.transaction_hash + ): continue - if transfer.from_address == net_search_info[2] and transfer.to_address == net_search_info[0]: + if ( + transfer.from_address == net_search_info[2] + and transfer.to_address == net_search_info[0] + ): return_transfers[index].amount += trace.inputs["amount"] return_transfers[index].trace_address = [-1] - if transfer.from_address == net_search_info[0] and transfer.to_address == net_search_info[2]: + if ( + transfer.from_address == net_search_info[0] + and transfer.to_address == net_search_info[2] + ): return_transfers[index].amount -= trace.inputs["amount"] return_transfers[index].trace_address = [-1] else: - return_transfers.append(Transfer( - block_number=trace.block_number, - transaction_hash=trace.transaction_hash, - trace_address=trace.trace_address, - from_address=net_search_info[2], # Janky... improve - to_address=net_search_info[0], - amount=trace.inputs["amount"], - token_address=net_search_info[1] - )) + return_transfers.append( + Transfer( + block_number=trace.block_number, + transaction_hash=trace.transaction_hash, + trace_address=trace.trace_address, + from_address=net_search_info[2], # Janky... improve + to_address=net_search_info[0], + amount=trace.inputs["amount"], + token_address=net_search_info[1], + ) + ) found_transfers.append(sorted(net_search_info)) i = 0 diff --git a/tests/test_jit_liquidity.py b/tests/test_jit_liquidity.py index 8d36662..fe5c58e 100644 --- a/tests/test_jit_liquidity.py +++ b/tests/test_jit_liquidity.py @@ -1,10 +1,9 @@ +from mev_inspect.classifiers.trace import TraceClassifier +from mev_inspect.jit_liquidity import get_jit_liquidity from mev_inspect.schemas.jit_liquidity import JITLiquidity from mev_inspect.schemas.swaps import Swap from mev_inspect.schemas.traces import Protocol from mev_inspect.swaps import get_swaps -from mev_inspect.jit_liquidity import get_jit_liquidity - -from mev_inspect.classifiers.trace import TraceClassifier from .utils import load_test_block @@ -32,7 +31,7 @@ def test_single_sandwich_jit_liquidity(trace_classifier: TraceClassifier): token_in_amount=1896817745609, token_out_address="0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2".lower(), token_out_amount=408818202022592862626, - protocol=Protocol.uniswap_v3 + protocol=Protocol.uniswap_v3, ) expected_jit_liquidity = [ JITLiquidity( @@ -58,20 +57,45 @@ def test_single_sandwich_jit_liquidity(trace_classifier: TraceClassifier): # Might be super janky but this could be done with assert jit_liquidity_instances == expected_jit_liquidity assert len(jit_liquidity_instances) == 1 assert len(jit_liquidity_instances[0].swaps) == 1 - assert jit_liquidity_instances[0].burn_transaction_hash == expected_jit_liquidity[0].burn_transaction_hash - assert jit_liquidity_instances[0].mint_transaction_hash == expected_jit_liquidity[0].mint_transaction_hash - assert jit_liquidity_instances[0].burn_token0_amount == expected_jit_liquidity[0].burn_token0_amount - assert jit_liquidity_instances[0].burn_token1_amount == expected_jit_liquidity[0].burn_token1_amount - assert jit_liquidity_instances[0].mint_token0_amount == expected_jit_liquidity[0].mint_token0_amount - assert jit_liquidity_instances[0].mint_token1_amount == expected_jit_liquidity[0].mint_token1_amount - assert jit_liquidity_instances[0].bot_address == expected_jit_liquidity[0].bot_address - assert jit_liquidity_instances[0].token0_swap_volume == expected_jit_liquidity[0].token0_swap_volume - assert jit_liquidity_instances[0].token1_swap_volume == expected_jit_liquidity[0].token1_swap_volume + assert ( + jit_liquidity_instances[0].burn_transaction_hash + == expected_jit_liquidity[0].burn_transaction_hash + ) + assert ( + jit_liquidity_instances[0].mint_transaction_hash + == expected_jit_liquidity[0].mint_transaction_hash + ) + assert ( + jit_liquidity_instances[0].burn_token0_amount + == expected_jit_liquidity[0].burn_token0_amount + ) + assert ( + jit_liquidity_instances[0].burn_token1_amount + == expected_jit_liquidity[0].burn_token1_amount + ) + assert ( + jit_liquidity_instances[0].mint_token0_amount + == expected_jit_liquidity[0].mint_token0_amount + ) + assert ( + jit_liquidity_instances[0].mint_token1_amount + == expected_jit_liquidity[0].mint_token1_amount + ) + assert ( + jit_liquidity_instances[0].bot_address == expected_jit_liquidity[0].bot_address + ) + assert ( + jit_liquidity_instances[0].token0_swap_volume + == expected_jit_liquidity[0].token0_swap_volume + ) + assert ( + jit_liquidity_instances[0].token1_swap_volume + == expected_jit_liquidity[0].token1_swap_volume + ) # Swap Checks - assert jit_liquidity_instances[0].swaps[0].transaction_hash == jit_swap.transaction_hash + assert ( + jit_liquidity_instances[0].swaps[0].transaction_hash + == jit_swap.transaction_hash + ) assert jit_liquidity_instances[0].swaps[0].trace_address == jit_swap.trace_address - - - -