331 lines
18 KiB
Python
331 lines
18 KiB
Python
import asyncio
|
|
import logging
|
|
import time
|
|
from typing import List, Optional
|
|
|
|
from sqlalchemy import orm
|
|
from web3 import Web3
|
|
|
|
from mev_inspect.fees import fetch_base_fee_per_gas
|
|
from mev_inspect.schemas.blocks import Block
|
|
from mev_inspect.schemas.receipts import Receipt
|
|
from mev_inspect.schemas.traces import Trace, TraceType
|
|
from mev_inspect.schemas.swaps import Swap
|
|
from mev_inspect.schemas.liquidations import Liquidation
|
|
|
|
from mev_inspect.utils import hex_to_int
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def get_latest_block_number(base_provider) -> int:
|
|
latest_block = await base_provider.make_request(
|
|
"eth_getBlockByNumber",
|
|
["latest", False],
|
|
)
|
|
|
|
return hex_to_int(latest_block["result"]["number"])
|
|
|
|
async def _get_logs_for_topics(w3, after_block, before_block, topics):
|
|
return await w3.eth.get_logs(
|
|
{
|
|
"fromBlock": hex(after_block),
|
|
"toBlock": hex(before_block),
|
|
"topics": topics,
|
|
})
|
|
|
|
|
|
def get_swap(data):
|
|
data = data[2:]
|
|
# print(data)
|
|
return int(data[0:64], base=16), int(data[64:128], base=16), int(data[128:192], base=16), int(data[192:256], base=16)
|
|
|
|
def get_liquidation(data):
|
|
data = data[2:]
|
|
# print(data)
|
|
return int(data[0:64], base=16), int(data[64:128], base=16), "0x"+data[128+24:168+24]
|
|
|
|
univ2abi = '''
|
|
[{"inputs":[],"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"owner","type":"address"},{"indexed":true,"internalType":"address","name":"spender","type":"address"},{"indexed":false,"internalType":"uint256","name":"value","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"sender","type":"address"},{"indexed":false,"internalType":"uint256","name":"amount0","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1","type":"uint256"},{"indexed":true,"internalType":"address","name":"to","type":"address"}],"name":"Burn","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"sender","type":"address"},{"indexed":false,"internalType":"uint256","name":"amount0","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1","type":"uint256"}],"name":"Mint","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"sender","type":"address"},{"indexed":false,"internalType":"uint256","name":"amount0In","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1In","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount0Out","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1Out","type":"uint256"},{"indexed":true,"internalType":"address","name":"to","type":"address"}],"name":"Swap","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"uint112","name":"reserve0","type":"uint112"},{"indexed":false,"internalType":"uint112","name":"reserve1","type":"uint112"}],"name":"Sync","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"from","type":"address"},{"indexed":true,"internalType":"address","name":"to","type":"address"},{"indexed":false,"internalType":"uint256","name":"value","type":"uint256"}],"name":"Transfer","type":"event"},{"inputs":[],"name":"DOMAIN_SEPARATOR","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"MINIMUM_LIQUIDITY","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"PERMIT_TYPEHASH","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"}],"name":"allowance","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"spender","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"}],"name":"approve","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"}],"name":"balanceOf","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"}],"name":"burn","outputs":[{"internalType":"uint256","name":"amount0","type":"uint256"},{"internalType":"uint256","name":"amount1","type":"uint256"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"decimals","outputs":[{"internalType":"uint8","name":"","type":"uint8"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"factory","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"getReserves","outputs":[{"internalType":"uint112","name":"_reserve0","type":"uint112"},{"internalType":"uint112","name":"_reserve1","type":"uint112"},{"internalType":"uint32","name":"_blockTimestampLast","type":"uint32"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_token0","type":"address"},{"internalType":"address","name":"_token1","type":"address"}],"name":"initialize","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"kLast","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"}],"name":"mint","outputs":[{"internalType":"uint256","name":"liquidity","type":"uint256"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"name","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"}],"name":"nonces","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"owner","type":"address"},{"internalType":"address","name":"spender","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"},{"internalType":"uint256","name":"deadline","type":"uint256"},{"internalType":"uint8","name":"v","type":"uint8"},{"internalType":"bytes32","name":"r","type":"bytes32"},{"internalType":"bytes32","name":"s","type":"bytes32"}],"name":"permit","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"price0CumulativeLast","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"price1CumulativeLast","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"}],"name":"skim","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"amount0Out","type":"uint256"},{"internalType":"uint256","name":"amount1Out","type":"uint256"},{"internalType":"address","name":"to","type":"address"},{"internalType":"bytes","name":"data","type":"bytes"}],"name":"swap","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"symbol","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"sync","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"token0","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"token1","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"totalSupply","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"}],"name":"transfer","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"from","type":"address"},{"internalType":"address","name":"to","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"}],"name":"transferFrom","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"}]
|
|
'''
|
|
|
|
def parse_topic(log, index):
|
|
return log['topics'][index].hex()
|
|
|
|
def parse_blockNumber(log):
|
|
return log['blockNumber']
|
|
|
|
def parse_transactionHash(log):
|
|
return log['transactionHash'].hex()
|
|
|
|
def parse_address(log):
|
|
return log['address'].lower()
|
|
|
|
def parse_logIndex(log):
|
|
return log['logIndex']
|
|
|
|
def parse_data(log):
|
|
return log['data']
|
|
|
|
def parse_token(w3, token):
|
|
return "0x"+w3.toHex(token)[26:]
|
|
|
|
async def classify_logs(logs, reserves, w3):
|
|
cswaps = []
|
|
cliquidations = []
|
|
new_reserves = []
|
|
topic_swap = "0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822"
|
|
topic_liquidation = "0xe413a321e8681d831f4dbccbca790d2952b56f977908e45be37335533e005286"
|
|
for log in logs:
|
|
if parse_topic(log, 0) == topic_swap:
|
|
block = parse_blockNumber(log)
|
|
transaction_hash = parse_transactionHash(log)
|
|
pool_address = parse_address(log)
|
|
if pool_address in reserves:
|
|
token0, token1 = reserves[pool_address]
|
|
else:
|
|
addr = Web3.toChecksumAddress(pool_address)
|
|
token0, token1 = await asyncio.gather(
|
|
w3.eth.call({'to': addr, 'data': '0x0dfe1681'}),
|
|
w3.eth.call({'to': addr, 'data': '0xd21220a7'})
|
|
)
|
|
token0 = parse_token(w3, token0)
|
|
token1 = parse_token(w3, token1)
|
|
reserves[pool_address] = (token0, token1)
|
|
new_reserves.append(
|
|
{
|
|
"pool_address": pool_address,
|
|
"token0": token0,
|
|
"token1": token1
|
|
}
|
|
)
|
|
|
|
am0in, am1in, am0out, am1out = get_swap(parse_data(log))
|
|
swap = Swap(
|
|
abi_name="uniswap_v2",
|
|
transaction_hash=transaction_hash,
|
|
block_number=block,
|
|
trace_address=[parse_logIndex(log)],
|
|
contract_address=pool_address,
|
|
from_address="0x"+parse_topic(log, 1)[26:],
|
|
to_address="0x"+parse_topic(log, 2)[26:],
|
|
token_in_address=token0 if am0in != 0 else token1, # TODO
|
|
token_in_amount= am0in if am0in != 0 else am1in,
|
|
token_out_address=token1 if am1out != 0 else token0, # TODO
|
|
token_out_amount= am0out if am0out != 0 else am1out,
|
|
protocol=None,
|
|
error=None
|
|
)
|
|
cswaps.append(swap)
|
|
elif parse_topic(log, 0) == topic_liquidation:
|
|
block = str(parse_blockNumber(log))
|
|
am_debt, am_recv, addr_usr = get_liquidation(parse_data(log))
|
|
liquidation = Liquidation(
|
|
liquidated_user = "0x"+parse_topic(log, 3)[26:],
|
|
liquidator_user = addr_usr,
|
|
debt_token_address = "0x"+parse_topic(log, 2)[26:],
|
|
debt_purchase_amount = am_debt,
|
|
received_amount = am_recv,
|
|
received_token_address = "0x"+parse_topic(log, 1)[26:],
|
|
protocol = None,
|
|
transaction_hash = parse_transactionHash(log),
|
|
trace_address = [parse_logIndex(log)],
|
|
block_number = block,
|
|
error = None
|
|
)
|
|
cliquidations.append(liquidation)
|
|
|
|
return cswaps, cliquidations, new_reserves
|
|
|
|
async def get_classified_traces_from_events(w3, after_block, before_block, reserves):
|
|
start = after_block
|
|
stride = 2000
|
|
topic_swap = "0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822"
|
|
topic_liquidation = "0xe413a321e8681d831f4dbccbca790d2952b56f977908e45be37335533e005286"
|
|
while start < before_block:
|
|
begin = start
|
|
end = start + stride if (start + stride) < before_block else before_block
|
|
end -= 1
|
|
start += stride
|
|
print("fetching from node...", begin, end, flush=True)
|
|
all_logs = await _get_logs_for_topics(w3, begin, end, [[topic_swap, topic_liquidation]])
|
|
yield await classify_logs(all_logs, reserves, w3)
|
|
|
|
async def create_from_block_number(
|
|
w3: Web3,
|
|
block_number: int,
|
|
trace_db_session: Optional[orm.Session],
|
|
) -> Block:
|
|
block_timestamp, receipts, traces, base_fee_per_gas = await asyncio.gather(
|
|
_find_or_fetch_block_timestamp(w3, block_number, trace_db_session),
|
|
_find_or_fetch_block_receipts(w3, block_number, trace_db_session),
|
|
_find_or_fetch_block_traces(w3, block_number, trace_db_session),
|
|
_find_or_fetch_base_fee_per_gas(w3, block_number, trace_db_session),
|
|
)
|
|
|
|
miner_address = _get_miner_address_from_traces(traces)
|
|
|
|
return Block(
|
|
block_number=block_number,
|
|
block_timestamp=block_timestamp,
|
|
miner=miner_address,
|
|
base_fee_per_gas=base_fee_per_gas,
|
|
traces=traces,
|
|
receipts=receipts,
|
|
)
|
|
|
|
|
|
async def _find_or_fetch_block_timestamp(
|
|
w3,
|
|
block_number: int,
|
|
trace_db_session: Optional[orm.Session],
|
|
) -> int:
|
|
if trace_db_session is not None:
|
|
existing_block_timestamp = _find_block_timestamp(trace_db_session, block_number)
|
|
if existing_block_timestamp is not None:
|
|
return existing_block_timestamp
|
|
|
|
return await _fetch_block_timestamp(w3, block_number)
|
|
|
|
|
|
async def _find_or_fetch_block_receipts(
|
|
w3,
|
|
block_number: int,
|
|
trace_db_session: Optional[orm.Session],
|
|
) -> List[Receipt]:
|
|
if trace_db_session is not None:
|
|
existing_block_receipts = _find_block_receipts(trace_db_session, block_number)
|
|
if existing_block_receipts is not None:
|
|
return existing_block_receipts
|
|
|
|
return await _fetch_block_receipts(w3, block_number)
|
|
|
|
|
|
async def _find_or_fetch_block_traces(
|
|
w3,
|
|
block_number: int,
|
|
trace_db_session: Optional[orm.Session],
|
|
) -> List[Trace]:
|
|
if trace_db_session is not None:
|
|
existing_block_traces = _find_block_traces(trace_db_session, block_number)
|
|
if existing_block_traces is not None:
|
|
return existing_block_traces
|
|
|
|
return await _fetch_block_traces(w3, block_number)
|
|
|
|
|
|
async def _find_or_fetch_base_fee_per_gas(
|
|
w3,
|
|
block_number: int,
|
|
trace_db_session: Optional[orm.Session],
|
|
) -> int:
|
|
if trace_db_session is not None:
|
|
existing_base_fee_per_gas = _find_base_fee_per_gas(
|
|
trace_db_session, block_number
|
|
)
|
|
if existing_base_fee_per_gas is not None:
|
|
return existing_base_fee_per_gas
|
|
|
|
return await fetch_base_fee_per_gas(w3, block_number)
|
|
|
|
|
|
async def _fetch_block_timestamp(w3, block_number: int) -> int:
|
|
block_json = await w3.eth.get_block(block_number)
|
|
return block_json["timestamp"]
|
|
|
|
|
|
async def _fetch_block_receipts(w3, block_number: int) -> List[Receipt]:
|
|
receipts_json = await w3.eth.get_block_receipts(block_number)
|
|
return [Receipt(**receipt) for receipt in receipts_json]
|
|
|
|
|
|
async def _fetch_block_traces(w3, block_number: int) -> List[Trace]:
|
|
traces_json = await w3.eth.trace_block(block_number)
|
|
return [Trace(**trace_json) for trace_json in traces_json]
|
|
|
|
|
|
def _find_block_timestamp(
|
|
trace_db_session: orm.Session,
|
|
block_number: int,
|
|
) -> Optional[int]:
|
|
result = trace_db_session.execute(
|
|
"SELECT block_timestamp FROM block_timestamps WHERE block_number = :block_number",
|
|
params={"block_number": block_number},
|
|
).one_or_none()
|
|
|
|
if result is None:
|
|
return None
|
|
else:
|
|
(block_timestamp,) = result
|
|
return block_timestamp
|
|
|
|
|
|
def _find_block_traces(
|
|
trace_db_session: orm.Session,
|
|
block_number: int,
|
|
) -> Optional[List[Trace]]:
|
|
result = trace_db_session.execute(
|
|
"SELECT raw_traces FROM block_traces WHERE block_number = :block_number",
|
|
params={"block_number": block_number},
|
|
).one_or_none()
|
|
|
|
if result is None:
|
|
return None
|
|
else:
|
|
(traces_json,) = result
|
|
return [Trace(**trace_json) for trace_json in traces_json]
|
|
|
|
|
|
def _find_block_receipts(
|
|
trace_db_session: orm.Session,
|
|
block_number: int,
|
|
) -> Optional[List[Receipt]]:
|
|
result = trace_db_session.execute(
|
|
"SELECT raw_receipts FROM block_receipts WHERE block_number = :block_number",
|
|
params={"block_number": block_number},
|
|
).one_or_none()
|
|
|
|
if result is None:
|
|
return None
|
|
else:
|
|
(receipts_json,) = result
|
|
return [Receipt(**receipt) for receipt in receipts_json]
|
|
|
|
|
|
def _find_base_fee_per_gas(
|
|
trace_db_session: orm.Session,
|
|
block_number: int,
|
|
) -> Optional[int]:
|
|
result = trace_db_session.execute(
|
|
"SELECT base_fee_in_wei FROM base_fee WHERE block_number = :block_number",
|
|
params={"block_number": block_number},
|
|
).one_or_none()
|
|
|
|
if result is None:
|
|
return None
|
|
else:
|
|
(base_fee,) = result
|
|
return base_fee
|
|
|
|
|
|
def _get_miner_address_from_traces(traces: List[Trace]) -> Optional[str]:
|
|
for trace in traces:
|
|
if trace.type == TraceType.reward:
|
|
return trace.action["author"]
|
|
|
|
return None
|
|
|
|
|
|
def get_transaction_hashes(calls: List[Trace]) -> List[str]:
|
|
result = []
|
|
|
|
for call in calls:
|
|
if call.type != TraceType.reward:
|
|
if (
|
|
call.transaction_hash is not None
|
|
and call.transaction_hash not in result
|
|
):
|
|
result.append(call.transaction_hash)
|
|
|
|
return result
|