2023-01-05 11:30:08 +00:00

471 lines
22 KiB
Python

import asyncio
import logging
from typing import List, Optional
from sqlalchemy import orm
from web3 import Web3
from mev_inspect.fees import fetch_base_fee_per_gas
from mev_inspect.schemas.blocks import Block
from mev_inspect.schemas.liquidations import Liquidation
from mev_inspect.schemas.receipts import Receipt
from mev_inspect.schemas.swaps import Swap
from mev_inspect.schemas.traces import Trace, TraceType
logger = logging.getLogger(__name__)
async def get_latest_block_number(w3) -> int:
return await w3.eth.get_block_number()
async def _get_logs_for_topics(w3, after_block, before_block, topics):
return await w3.eth.get_logs(
{
"fromBlock": hex(after_block),
"toBlock": hex(before_block),
"topics": topics,
}
)
def twos_complement(hexstr, bits):
value = int(hexstr, 16)
if value & (1 << (bits - 1)):
value -= 1 << bits
return value
def get_swap_v2(data):
data = data[2:]
# print(data)
return (
int(data[0:64], base=16),
int(data[64:128], base=16),
int(data[128:192], base=16),
int(data[192:256], base=16),
)
def get_swap_v3(data):
data = data[2:]
# print(data)
return (
twos_complement(data[0:64], 256),
twos_complement(data[64:128], 256),
int(data[152:192], base=16),
int(data[224:256], base=16),
twos_complement(data[314:320], 24),
)
def get_swap_synapse(data):
data = data[2:]
# print(data)
return (
int(data[0:64], base=16),
int(data[64:128], base=16),
data[128:192],
data[192:256],
)
def get_liquidation(data):
data = data[2:]
# print(data)
return (
int(data[0:64], base=16),
int(data[64:128], base=16),
"0x" + data[128 + 24 : 168 + 24],
)
univ2abi = """
[{"inputs":[],"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"owner","type":"address"},{"indexed":true,"internalType":"address","name":"spender","type":"address"},{"indexed":false,"internalType":"uint256","name":"value","type":"uint256"}],"name":"Approval","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"sender","type":"address"},{"indexed":false,"internalType":"uint256","name":"amount0","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1","type":"uint256"},{"indexed":true,"internalType":"address","name":"to","type":"address"}],"name":"Burn","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"sender","type":"address"},{"indexed":false,"internalType":"uint256","name":"amount0","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1","type":"uint256"}],"name":"Mint","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"sender","type":"address"},{"indexed":false,"internalType":"uint256","name":"amount0In","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1In","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount0Out","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"amount1Out","type":"uint256"},{"indexed":true,"internalType":"address","name":"to","type":"address"}],"name":"Swap","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"uint112","name":"reserve0","type":"uint112"},{"indexed":false,"internalType":"uint112","name":"reserve1","type":"uint112"}],"name":"Sync","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"from","type":"address"},{"indexed":true,"internalType":"address","name":"to","type":"address"},{"indexed":false,"internalType":"uint256","name":"value","type":"uint256"}],"name":"Transfer","type":"event"},{"inputs":[],"name":"DOMAIN_SEPARATOR","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"MINIMUM_LIQUIDITY","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"PERMIT_TYPEHASH","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"},{"internalType":"address","name":"","type":"address"}],"name":"allowance","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"spender","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"}],"name":"approve","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"}],"name":"balanceOf","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"}],"name":"burn","outputs":[{"internalType":"uint256","name":"amount0","type":"uint256"},{"internalType":"uint256","name":"amount1","type":"uint256"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"decimals","outputs":[{"internalType":"uint8","name":"","type":"uint8"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"factory","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"getReserves","outputs":[{"internalType":"uint112","name":"_reserve0","type":"uint112"},{"internalType":"uint112","name":"_reserve1","type":"uint112"},{"internalType":"uint32","name":"_blockTimestampLast","type":"uint32"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_token0","type":"address"},{"internalType":"address","name":"_token1","type":"address"}],"name":"initialize","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"kLast","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"}],"name":"mint","outputs":[{"internalType":"uint256","name":"liquidity","type":"uint256"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"name","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"","type":"address"}],"name":"nonces","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"owner","type":"address"},{"internalType":"address","name":"spender","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"},{"internalType":"uint256","name":"deadline","type":"uint256"},{"internalType":"uint8","name":"v","type":"uint8"},{"internalType":"bytes32","name":"r","type":"bytes32"},{"internalType":"bytes32","name":"s","type":"bytes32"}],"name":"permit","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"price0CumulativeLast","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"price1CumulativeLast","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"}],"name":"skim","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"amount0Out","type":"uint256"},{"internalType":"uint256","name":"amount1Out","type":"uint256"},{"internalType":"address","name":"to","type":"address"},{"internalType":"bytes","name":"data","type":"bytes"}],"name":"swap","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"symbol","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"sync","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"token0","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"token1","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"totalSupply","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"to","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"}],"name":"transfer","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"from","type":"address"},{"internalType":"address","name":"to","type":"address"},{"internalType":"uint256","name":"value","type":"uint256"}],"name":"transferFrom","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"}]
"""
def parse_topic(log, index):
return log["topics"][index].hex()
def parse_blockNumber(log):
return log["blockNumber"]
def parse_transactionHash(log):
return log["transactionHash"].hex()
def parse_address(log):
return log["address"].lower()
def parse_logIndex(log):
return log["logIndex"]
def parse_data(log):
return log["data"]
def parse_token(w3, token):
return "0x" + w3.toHex(token)[26:]
async def classify_logs(logs, reserves, w3):
cswaps = []
cliquidations = []
new_reserves = []
topic_swap_v2 = "0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822"
topic_swap_v3 = "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67"
topic_swap_synapse = (
"0xc6c1e0630dbe9130cc068028486c0d118ddcea348550819defd5cb8c257f8a38"
)
topic_liquidation = (
"0xe413a321e8681d831f4dbccbca790d2952b56f977908e45be37335533e005286"
)
for log in logs:
if parse_topic(log, 0) == topic_swap_v2:
block = parse_blockNumber(log)
transaction_hash = parse_transactionHash(log)
pool_address = parse_address(log)
if pool_address in reserves:
token0, token1 = reserves[pool_address]
else:
addr = Web3.toChecksumAddress(pool_address)
token0, token1 = await asyncio.gather(
w3.eth.call({"to": addr, "data": "0x0dfe1681"}),
w3.eth.call({"to": addr, "data": "0xd21220a7"}),
)
token0 = parse_token(w3, token0)
token1 = parse_token(w3, token1)
reserves[pool_address] = (token0, token1)
new_reserves.append(
{"pool_address": pool_address, "token0": token0, "token1": token1}
)
am0in, am1in, am0out, am1out = get_swap_v2(parse_data(log))
swap = Swap(
abi_name="uniswap_v2",
transaction_hash=transaction_hash,
block_number=block,
trace_address=[parse_logIndex(log)],
contract_address=pool_address,
from_address="0x" + parse_topic(log, 1)[26:],
to_address="0x" + parse_topic(log, 2)[26:],
token_in_address=token0 if am0in != 0 else token1, # TODO
token_in_amount=am0in if am0in != 0 else am1in,
token_out_address=token1 if am1out != 0 else token0, # TODO
token_out_amount=am0out if am0out != 0 else am1out,
protocol=None,
error=None,
)
cswaps.append(swap)
elif parse_topic(log, 0) == topic_swap_v3:
block = parse_blockNumber(log)
transaction_hash = parse_transactionHash(log)
pool_address = parse_address(log)
if pool_address in reserves:
token0, token1 = reserves[pool_address]
else:
addr = Web3.toChecksumAddress(pool_address)
token0, token1 = await asyncio.gather(
w3.eth.call({"to": addr, "data": "0x0dfe1681"}),
w3.eth.call({"to": addr, "data": "0xd21220a7"}),
)
token0 = parse_token(w3, token0)
token1 = parse_token(w3, token1)
reserves[pool_address] = (token0, token1)
new_reserves.append(
{"pool_address": pool_address, "token0": token0, "token1": token1}
)
am0, am1, _, _, _ = get_swap_v3(parse_data(log))
swap = Swap(
abi_name="uniswap_v3",
transaction_hash=transaction_hash,
block_number=block,
trace_address=[parse_logIndex(log)],
contract_address=pool_address,
from_address="0x" + parse_topic(log, 1)[26:],
to_address="0x" + parse_topic(log, 2)[26:],
token_in_address=token0 if am0 > 0 else token1, # TODO
token_in_amount=am0 if am0 > 0 else am1,
token_out_address=token1 if am1 < 0 else token0, # TODO
token_out_amount=-am1 if am1 < 0 else -am0,
protocol=None,
error=None,
)
cswaps.append(swap)
elif parse_topic(log, 0) == topic_swap_synapse:
block = parse_blockNumber(log)
transaction_hash = parse_transactionHash(log)
pool_address = parse_address(log)
sold, bought, soldId, boughtId = get_swap_synapse(parse_data(log))
if pool_address in reserves:
pass # TODO
else:
addr = Web3.toChecksumAddress(pool_address)
soldToken, boughtToken = await asyncio.gather(
w3.eth.call({"to": addr, "data": "0x82b86600" + soldId}),
w3.eth.call({"to": addr, "data": "0x82b86600" + boughtId}),
)
soldToken = parse_token(w3, soldToken)
boughtToken = parse_token(w3, boughtToken)
# reserves[pool_address] = (token0, token1)
# new_reserves.append(
# {
# "pool_address": pool_address,
# "token0": token0,
# "token1": token1
# }
# )
swap = Swap(
abi_name="synapse",
transaction_hash=transaction_hash,
block_number=block,
trace_address=[parse_logIndex(log)],
contract_address=pool_address,
from_address="0x" + parse_topic(log, 1)[26:],
to_address="0x" + parse_topic(log, 1)[26:],
token_in_address=soldToken,
token_in_amount=sold,
token_out_address=boughtToken,
token_out_amount=bought,
protocol=None,
error=None,
)
cswaps.append(swap)
elif parse_topic(log, 0) == topic_liquidation:
block = str(parse_blockNumber(log))
am_debt, am_recv, addr_usr = get_liquidation(parse_data(log))
liquidation = Liquidation(
liquidated_user="0x" + parse_topic(log, 3)[26:],
liquidator_user=addr_usr,
debt_token_address="0x" + parse_topic(log, 2)[26:],
debt_purchase_amount=am_debt,
received_amount=am_recv,
received_token_address="0x" + parse_topic(log, 1)[26:],
protocol=None,
transaction_hash=parse_transactionHash(log),
trace_address=[parse_logIndex(log)],
block_number=block,
error=None,
)
cliquidations.append(liquidation)
return cswaps, cliquidations, new_reserves
async def get_classified_traces_from_events(w3, after_block, before_block, reserves):
start = after_block
stride = 2000
topic_swap_v2 = "0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822"
topic_swap_v3 = "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67"
topic_swap_synapse = (
"0xc6c1e0630dbe9130cc068028486c0d118ddcea348550819defd5cb8c257f8a38"
)
topic_liquidation = (
"0xe413a321e8681d831f4dbccbca790d2952b56f977908e45be37335533e005286"
)
while start < before_block:
begin = start
end = start + stride if (start + stride) < before_block else before_block
end -= 1
start += stride
logger.info("fetching from node... {} {}".format(begin, end))
all_logs = await _get_logs_for_topics(
w3,
begin,
end,
[[topic_swap_v2, topic_swap_v3, topic_swap_synapse, topic_liquidation]],
)
yield await classify_logs(all_logs, reserves, w3)
async def create_from_block_number(
w3: Web3,
block_number: int,
trace_db_session: Optional[orm.Session],
) -> Block:
block_timestamp, receipts, traces, base_fee_per_gas = await asyncio.gather(
_find_or_fetch_block_timestamp(w3, block_number, trace_db_session),
_find_or_fetch_block_receipts(w3, block_number, trace_db_session),
_find_or_fetch_block_traces(w3, block_number, trace_db_session),
_find_or_fetch_base_fee_per_gas(w3, block_number, trace_db_session),
)
miner_address = _get_miner_address_from_traces(traces)
return Block(
block_number=block_number,
block_timestamp=block_timestamp,
miner=miner_address,
base_fee_per_gas=base_fee_per_gas,
traces=traces,
receipts=receipts,
)
async def _find_or_fetch_block_timestamp(
w3,
block_number: int,
trace_db_session: Optional[orm.Session],
) -> int:
if trace_db_session is not None:
existing_block_timestamp = _find_block_timestamp(trace_db_session, block_number)
if existing_block_timestamp is not None:
return existing_block_timestamp
return await _fetch_block_timestamp(w3, block_number)
async def _find_or_fetch_block_receipts(
w3,
block_number: int,
trace_db_session: Optional[orm.Session],
) -> List[Receipt]:
if trace_db_session is not None:
existing_block_receipts = _find_block_receipts(trace_db_session, block_number)
if existing_block_receipts is not None:
return existing_block_receipts
return await _fetch_block_receipts(w3, block_number)
async def _find_or_fetch_block_traces(
w3,
block_number: int,
trace_db_session: Optional[orm.Session],
) -> List[Trace]:
if trace_db_session is not None:
existing_block_traces = _find_block_traces(trace_db_session, block_number)
if existing_block_traces is not None:
return existing_block_traces
return await _fetch_block_traces(w3, block_number)
async def _find_or_fetch_base_fee_per_gas(
w3,
block_number: int,
trace_db_session: Optional[orm.Session],
) -> int:
if trace_db_session is not None:
existing_base_fee_per_gas = _find_base_fee_per_gas(
trace_db_session, block_number
)
if existing_base_fee_per_gas is not None:
return existing_base_fee_per_gas
return await fetch_base_fee_per_gas(w3, block_number)
async def _fetch_block_timestamp(w3, block_number: int) -> int:
block_json = await w3.eth.get_block(block_number)
return block_json["timestamp"]
async def _fetch_block_receipts(w3, block_number: int) -> List[Receipt]:
receipts_json = await w3.eth.get_block_receipts(block_number)
return [Receipt(**receipt) for receipt in receipts_json]
async def _fetch_block_traces(w3, block_number: int) -> List[Trace]:
traces_json = await w3.eth.trace_block(block_number)
return [Trace(**trace_json) for trace_json in traces_json]
def _find_block_timestamp(
trace_db_session: orm.Session,
block_number: int,
) -> Optional[int]:
result = trace_db_session.execute(
"SELECT block_timestamp FROM block_timestamps WHERE block_number = :block_number",
params={"block_number": block_number},
).one_or_none()
if result is None:
return None
else:
(block_timestamp,) = result
return block_timestamp
def _find_block_traces(
trace_db_session: orm.Session,
block_number: int,
) -> Optional[List[Trace]]:
result = trace_db_session.execute(
"SELECT raw_traces FROM block_traces WHERE block_number = :block_number",
params={"block_number": block_number},
).one_or_none()
if result is None:
return None
else:
(traces_json,) = result
return [Trace(**trace_json) for trace_json in traces_json]
def _find_block_receipts(
trace_db_session: orm.Session,
block_number: int,
) -> Optional[List[Receipt]]:
result = trace_db_session.execute(
"SELECT raw_receipts FROM block_receipts WHERE block_number = :block_number",
params={"block_number": block_number},
).one_or_none()
if result is None:
return None
else:
(receipts_json,) = result
return [Receipt(**receipt) for receipt in receipts_json]
def _find_base_fee_per_gas(
trace_db_session: orm.Session,
block_number: int,
) -> Optional[int]:
result = trace_db_session.execute(
"SELECT base_fee_in_wei FROM base_fee WHERE block_number = :block_number",
params={"block_number": block_number},
).one_or_none()
if result is None:
return None
else:
(base_fee,) = result
return base_fee
def _get_miner_address_from_traces(traces: List[Trace]) -> Optional[str]:
for trace in traces:
if trace.type == TraceType.reward:
return trace.action["author"]
return None
def get_transaction_hashes(calls: List[Trace]) -> List[str]:
result = []
for call in calls:
if call.type != TraceType.reward:
if (
call.transaction_hash is not None
and call.transaction_hash not in result
):
result.append(call.transaction_hash)
return result