From cac1b13ac708589f43c52980dc06918d7e12425a Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Wed, 29 Sep 2021 10:24:46 -0400 Subject: [PATCH 1/5] Database migration for liquidations --- .../c8363617aa07_create_liquidations_table.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 alembic/versions/c8363617aa07_create_liquidations_table.py diff --git a/alembic/versions/c8363617aa07_create_liquidations_table.py b/alembic/versions/c8363617aa07_create_liquidations_table.py new file mode 100644 index 0000000..818858d --- /dev/null +++ b/alembic/versions/c8363617aa07_create_liquidations_table.py @@ -0,0 +1,38 @@ +"""Create liquidations table + +Revision ID: c8363617aa07 +Revises: cd96af55108e +Create Date: 2021-09-29 14:00:06.857103 + +""" +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "c8363617aa07" +down_revision = "cd96af55108e" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "liquidations", + sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()), + sa.Column("liquidated_user", sa.String(256), nullable=False), + sa.Column("liquidator_user", sa.String(256), nullable=False), + sa.Column("collateral_token_address", sa.String(256), nullable=False), + sa.Column("debt_token_address", sa.String(256), nullable=False), + sa.Column("debt_purchase_amount", sa.Numeric, nullable=False), + sa.Column("received_amount", sa.Numeric, nullable=False), + sa.Column("protocol", sa.String(256), nullable=True), + sa.Column("transaction_hash", sa.String(66), nullable=False), + sa.Column("trace_address", sa.String(256), nullable=False), + sa.Column("block_number", sa.Numeric, nullable=False), + sa.PrimaryKeyConstraint("transaction_hash", "trace_address"), + ) + + +def downgrade(): + op.drop_table("liquidations") From 1560ee9a9923629a61bba541e93c64ff6f8b7d0a Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Wed, 29 Sep 2021 10:41:49 -0400 Subject: [PATCH 2/5] Add liquidations model/crud --- mev_inspect/crud/liquidations.py | 31 ++++++++++++++++++++++++++++++ mev_inspect/inspect_block.py | 10 ++++++++++ mev_inspect/models/liquidations.py | 18 +++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 mev_inspect/crud/liquidations.py create mode 100644 mev_inspect/models/liquidations.py diff --git a/mev_inspect/crud/liquidations.py b/mev_inspect/crud/liquidations.py new file mode 100644 index 0000000..2a58ead --- /dev/null +++ b/mev_inspect/crud/liquidations.py @@ -0,0 +1,31 @@ +import json +from typing import List + +from mev_inspect.models.liquidations import LiquidationModel +from mev_inspect.schemas.liquidations import Liquidation + + +def delete_liquidations_for_block( + db_session, + block_number: int, +) -> None: + ( + db_session.query(LiquidationModel) + .filter(LiquidationModel.block_number == block_number) + .delete() + ) + + db_session.commit() + + +def write_liquidations( + db_session, + liquidations: List[Liquidation], +) -> None: + models = [ + LiquidationModel(**json.loads(liquidation.json())) + for liquidation in liquidations + ] + + db_session.bulk_save_objects(models) + db_session.commit() diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py index f457f56..597e2db 100644 --- a/mev_inspect/inspect_block.py +++ b/mev_inspect/inspect_block.py @@ -17,8 +17,13 @@ from mev_inspect.crud.miner_payments import ( delete_miner_payments_for_block, write_miner_payments, ) + from mev_inspect.crud.swaps import delete_swaps_for_block, write_swaps from mev_inspect.crud.transfers import delete_transfers_for_block, write_transfers +from mev_inspect.crud.liquidations import ( + delete_liquidations_for_block, + write_liquidations, +) from mev_inspect.miner_payments import get_miner_payments from mev_inspect.swaps import get_swaps from mev_inspect.transfers import get_transfers @@ -38,6 +43,7 @@ def inspect_block( should_write_swaps: bool = True, should_write_transfers: bool = True, should_write_arbitrages: bool = True, + should_write_liquidations: bool = True, should_write_miner_payments: bool = True, ): block = create_from_block_number(base_provider, w3, block_number, should_cache) @@ -79,6 +85,10 @@ def inspect_block( liquidations = get_aave_liquidations(classified_traces) logger.info(f"Found {len(liquidations)} liquidations") + if should_write_liquidations: + delete_liquidations_for_block(db_session, block_number) + write_liquidations(db_session, liquidations) + miner_payments = get_miner_payments( block.miner, block.base_fee_per_gas, classified_traces, block.receipts ) diff --git a/mev_inspect/models/liquidations.py b/mev_inspect/models/liquidations.py new file mode 100644 index 0000000..8c58c04 --- /dev/null +++ b/mev_inspect/models/liquidations.py @@ -0,0 +1,18 @@ +from sqlalchemy import Column, Numeric, String, ARRAY, Integer + +from .base import Base + + +class LiquidationModel(Base): + __tablename__ = "liquidations" + + liquidated_user = Column(String, nullable=False) + liquidator_user = Column(String, nullable=False) + collateral_token_address = Column(String, nullable=False) + debt_token_address = Column(String, nullable=False) + debt_purchase_amount = Column(Numeric, nullable=False) + received_amount = Column(Numeric, nullable=False) + protocol = Column(String, nullable=True) + transaction_hash = Column(String, primary_key=True) + trace_address = Column(ARRAY(Integer), nullable=False) + block_number = Column(Numeric, nullable=False) From 4dbcb59b4d8198fc2d6a8bfcb6f34f371a4faa5b Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Wed, 29 Sep 2021 10:52:58 -0400 Subject: [PATCH 3/5] Add trace address to liquidations --- mev_inspect/aave_liquidations.py | 3 ++- mev_inspect/schemas/liquidations.py | 2 ++ tests/liquidation_test.py | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/mev_inspect/aave_liquidations.py b/mev_inspect/aave_liquidations.py index 34d0b75..3ed5df5 100644 --- a/mev_inspect/aave_liquidations.py +++ b/mev_inspect/aave_liquidations.py @@ -63,10 +63,11 @@ def get_aave_liquidations( protocol=Protocol.aave, received_amount=received_amount, transaction_hash=trace.transaction_hash, + trace_address=trace.trace_address, block_number=trace.block_number, ) ) - + print(liquidations) return liquidations diff --git a/mev_inspect/schemas/liquidations.py b/mev_inspect/schemas/liquidations.py index 0f8b183..fad090b 100644 --- a/mev_inspect/schemas/liquidations.py +++ b/mev_inspect/schemas/liquidations.py @@ -1,3 +1,4 @@ +from typing import List from pydantic import BaseModel from mev_inspect.schemas.classified_traces import Protocol @@ -11,4 +12,5 @@ class Liquidation(BaseModel): received_amount: int protocol: Protocol transaction_hash: str + trace_address: List[int] block_number: str diff --git a/tests/liquidation_test.py b/tests/liquidation_test.py index 83b2458..62c58e4 100644 --- a/tests/liquidation_test.py +++ b/tests/liquidation_test.py @@ -24,6 +24,7 @@ def test_single_weth_liquidation(): received_amount=8182733924513576561, protocol=Protocol.aave, transaction_hash=transaction_hash, + trace_address=[1, 1, 6], block_number=block_number, ) ] @@ -53,6 +54,7 @@ def test_single_liquidation(): received_amount=2657946947610159065393, protocol=Protocol.aave, transaction_hash=transaction_hash, + trace_address=[0, 7, 1, 0, 6], block_number=block_number, ) ] @@ -81,6 +83,7 @@ def test_multiple_liquidations_in_block(): received_amount=10111753901939162887, protocol=Protocol.aave, transaction_hash=transaction1, + trace_address=[], block_number=block_number, ) @@ -93,6 +96,7 @@ def test_multiple_liquidations_in_block(): received_amount=21996356316098208090, protocol=Protocol.aave, transaction_hash=transaction2, + trace_address=[], block_number=block_number, ) @@ -105,6 +109,7 @@ def test_multiple_liquidations_in_block(): received_amount=121531358145247546, protocol=Protocol.aave, transaction_hash=transaction3, + trace_address=[], block_number=block_number, ) From eb720dee1600d0b456453a746e17e4389193a8b2 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Wed, 29 Sep 2021 10:54:43 -0400 Subject: [PATCH 4/5] Remove print --- mev_inspect/aave_liquidations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mev_inspect/aave_liquidations.py b/mev_inspect/aave_liquidations.py index 3ed5df5..49e4c3c 100644 --- a/mev_inspect/aave_liquidations.py +++ b/mev_inspect/aave_liquidations.py @@ -67,7 +67,6 @@ def get_aave_liquidations( block_number=trace.block_number, ) ) - print(liquidations) return liquidations From 77b17cab94a65088b592607cef5aaeaf68a3c81e Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Fri, 1 Oct 2021 17:59:11 -0400 Subject: [PATCH 5/5] Make trace address a primary key --- mev_inspect/models/liquidations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mev_inspect/models/liquidations.py b/mev_inspect/models/liquidations.py index 8c58c04..1c020a9 100644 --- a/mev_inspect/models/liquidations.py +++ b/mev_inspect/models/liquidations.py @@ -14,5 +14,5 @@ class LiquidationModel(Base): received_amount = Column(Numeric, nullable=False) protocol = Column(String, nullable=True) transaction_hash = Column(String, primary_key=True) - trace_address = Column(ARRAY(Integer), nullable=False) + trace_address = Column(ARRAY(Integer), primary_key=True) block_number = Column(Numeric, nullable=False)