From acb68dfcff7e52a83ac954750227d02abe24faf7 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 30 Aug 2021 13:45:34 -0400 Subject: [PATCH 1/6] Add miner_payments table --- ...83978d6e455_create_miner_payments_table.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 alembic/versions/083978d6e455_create_miner_payments_table.py diff --git a/alembic/versions/083978d6e455_create_miner_payments_table.py b/alembic/versions/083978d6e455_create_miner_payments_table.py new file mode 100644 index 0000000..e55d3a8 --- /dev/null +++ b/alembic/versions/083978d6e455_create_miner_payments_table.py @@ -0,0 +1,36 @@ +"""Create miner_payments table + +Revision ID: 083978d6e455 +Revises: 92f28a2b4f52 +Create Date: 2021-08-30 17:42:25.548130 + +""" +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "083978d6e455" +down_revision = "92f28a2b4f52" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "miner_payments", + sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()), + sa.Column("block_number", sa.Numeric, nullable=False), + sa.Column("transaction_hash", sa.String(66), primary_key=True), + sa.Column("transaction_index", sa.Numeric, nullable=False), + sa.Column("miner_address", sa.String(256), nullable=False), + sa.Column("coinbase_transfer", sa.Numeric, nullable=False), + sa.Column("base_fee_per_gas", sa.Numeric, nullable=False), + sa.Column("gas_price", sa.Numeric, nullable=False), + sa.Column("gas_price_with_coinbase_transfer", sa.Numeric, nullable=False), + sa.Column("gas_used", sa.Numeric, nullable=False), + ) + + +def downgrade(): + op.drop_table("miner_payments") From c22cab0b684e6ccad2d79be6ec4b99dcc83c10e2 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 30 Aug 2021 13:47:37 -0400 Subject: [PATCH 2/6] Add miner_payments model --- mev_inspect/models/miner_payments.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 mev_inspect/models/miner_payments.py diff --git a/mev_inspect/models/miner_payments.py b/mev_inspect/models/miner_payments.py new file mode 100644 index 0000000..136130e --- /dev/null +++ b/mev_inspect/models/miner_payments.py @@ -0,0 +1,17 @@ +from sqlalchemy import Column, Numeric, String + +from .base import Base + + +class SwapModel(Base): + __tablename__ = "swaps" + + block_number = Column(Numeric, nullable=False) + transaction_hash = Column(String, primary_key=True) + transaction_index = Column(Numeric, nullable=False) + miner_address = Column(String, nullable=False) + coinbase_transfer = Column(Numeric, nullable=False) + base_fee_per_gas = Column(Numeric, nullable=False) + gas_price = Column(Numeric, nullable=False) + gas_price_with_coinbase_transfer = Column(Numeric, nullable=False) + gas_used = Column(Numeric, nullable=False) From f5222c1902cf7c412c5674fffda6b832053c4a14 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 30 Aug 2021 13:48:35 -0400 Subject: [PATCH 3/6] Swap => MinerPayment --- mev_inspect/models/miner_payments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mev_inspect/models/miner_payments.py b/mev_inspect/models/miner_payments.py index 136130e..b3c1e68 100644 --- a/mev_inspect/models/miner_payments.py +++ b/mev_inspect/models/miner_payments.py @@ -3,8 +3,8 @@ from sqlalchemy import Column, Numeric, String from .base import Base -class SwapModel(Base): - __tablename__ = "swaps" +class MinerPaymentModel(Base): + __tablename__ = "miner_payments" block_number = Column(Numeric, nullable=False) transaction_hash = Column(String, primary_key=True) From 9769498c9d204a1998396a677f4c5b72276e093e Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 30 Aug 2021 13:49:40 -0400 Subject: [PATCH 4/6] Add crud for miner payments --- mev_inspect/crud/miner_payments.py | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 mev_inspect/crud/miner_payments.py diff --git a/mev_inspect/crud/miner_payments.py b/mev_inspect/crud/miner_payments.py new file mode 100644 index 0000000..e82bf96 --- /dev/null +++ b/mev_inspect/crud/miner_payments.py @@ -0,0 +1,31 @@ +import json +from typing import List + +from mev_inspect.models.miner_payments import MinerPaymentModel +from mev_inspect.schemas.miner_payments import MinerPayment + + +def delete_miner_payments_for_block( + db_session, + block_number: int, +) -> None: + ( + db_session.query(MinerPaymentModel) + .filter(MinerPaymentModel.block_number == block_number) + .delete() + ) + + db_session.commit() + + +def write_miner_payments( + db_session, + miner_payments: List[MinerPayment], +) -> None: + models = [ + MinerPaymentModel(**json.loads(miner_payment.json())) + for miner_payment in miner_payments + ] + + db_session.bulk_save_objects(models) + db_session.commit() From 1aa5c5b4657466da70e6ed597efe1f038a3810f1 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 30 Aug 2021 18:02:36 -0400 Subject: [PATCH 5/6] Write miner payments to the DB in script run --- scripts/inspect_block.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/inspect_block.py b/scripts/inspect_block.py index 1712947..2f798d2 100644 --- a/scripts/inspect_block.py +++ b/scripts/inspect_block.py @@ -3,7 +3,10 @@ import json import click from web3 import Web3 +from mev_inspect.arbitrages import get_arbitrages from mev_inspect.block import create_from_block_number +from mev_inspect.classifiers.specs import ALL_CLASSIFIER_SPECS +from mev_inspect.classifiers.trace import TraceClassifier from mev_inspect.crud.arbitrages import ( delete_arbitrages_for_block, write_arbitrages, @@ -12,10 +15,10 @@ from mev_inspect.crud.classified_traces import ( delete_classified_traces_for_block, write_classified_traces, ) - -from mev_inspect.arbitrages import get_arbitrages -from mev_inspect.classifiers.specs import ALL_CLASSIFIER_SPECS -from mev_inspect.classifiers.trace import TraceClassifier +from mev_inspect.crud.miner_payments import ( + delete_miner_payments_for_block, + write_miner_payments, +) from mev_inspect.crud.swaps import delete_swaps_for_block, write_swaps from mev_inspect.db import get_session from mev_inspect.miner_payments import get_miner_payments @@ -78,10 +81,11 @@ def _inspect_block( block_number: int, should_cache: bool, should_print_stats: bool = True, + should_print_miner_payments: bool = True, should_write_classified_traces: bool = True, should_write_swaps: bool = True, should_write_arbitrages: bool = True, - should_print_miner_payments: bool = True, + should_write_miner_payments: bool = True, ): block = create_from_block_number(base_provider, w3, block_number, should_cache) @@ -127,6 +131,10 @@ def _inspect_block( if should_print_miner_payments: click.echo(json.dumps([p.dict() for p in miner_payments], indent=4)) + if should_write_miner_payments: + delete_miner_payments_for_block(db_session, block_number) + write_miner_payments(db_session, miner_payments) + def get_stats(classified_traces) -> dict: stats: dict = {} From 9846b9f16ce789a68d5bf527ce80241a5c87a4f0 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 30 Aug 2021 20:09:01 -0400 Subject: [PATCH 6/6] Add to and from addresses --- ...b6f_add_to_address_and_from_address_to_.py | 32 +++++++++++++++++++ mev_inspect/miner_payments.py | 8 +++++ mev_inspect/models/miner_payments.py | 2 ++ mev_inspect/schemas/miner_payments.py | 4 +++ 4 files changed, 46 insertions(+) create mode 100644 alembic/versions/d70c08b4db6f_add_to_address_and_from_address_to_.py diff --git a/alembic/versions/d70c08b4db6f_add_to_address_and_from_address_to_.py b/alembic/versions/d70c08b4db6f_add_to_address_and_from_address_to_.py new file mode 100644 index 0000000..01f075c --- /dev/null +++ b/alembic/versions/d70c08b4db6f_add_to_address_and_from_address_to_.py @@ -0,0 +1,32 @@ +"""Add to_address and from_address to miner_payments table + +Revision ID: d70c08b4db6f +Revises: 083978d6e455 +Create Date: 2021-08-30 22:10:04.186251 + +""" +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "d70c08b4db6f" +down_revision = "083978d6e455" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "miner_payments", + sa.Column("transaction_to_address", sa.String(256), nullable=True), + ) + op.add_column( + "miner_payments", + sa.Column("transaction_from_address", sa.String(256), nullable=True), + ) + + +def downgrade(): + op.drop_column("miner_payments", "transaction_to_address") + op.drop_column("miner_payments", "transaction_from_address") diff --git a/mev_inspect/miner_payments.py b/mev_inspect/miner_payments.py index e4fbeef..cb3d80f 100644 --- a/mev_inspect/miner_payments.py +++ b/mev_inspect/miner_payments.py @@ -22,6 +22,12 @@ def get_miner_payments( for receipt in receipts: transaciton_traces = traces_by_transaction_hash[receipt.transaction_hash] + + if len(transaciton_traces) == 0: + continue + + first_trace = sorted(transaciton_traces, key=lambda t: t.trace_address)[0] + eth_transfers = get_eth_transfers(transaciton_traces) miner_eth_transfers = filter_transfers( eth_transfers, to_address=miner_address.lower() @@ -46,6 +52,8 @@ def get_miner_payments( base_fee_per_gas=base_fee_per_gas, gas_used=receipt.gas_used, coinbase_transfer=coinbase_transfer, + transaction_to_address=first_trace.to_address, + transaction_from_address=first_trace.from_address, ) ) diff --git a/mev_inspect/models/miner_payments.py b/mev_inspect/models/miner_payments.py index b3c1e68..1336060 100644 --- a/mev_inspect/models/miner_payments.py +++ b/mev_inspect/models/miner_payments.py @@ -15,3 +15,5 @@ class MinerPaymentModel(Base): gas_price = Column(Numeric, nullable=False) gas_price_with_coinbase_transfer = Column(Numeric, nullable=False) gas_used = Column(Numeric, nullable=False) + transaction_from_address = Column(String, nullable=True) + transaction_to_address = Column(String, nullable=True) diff --git a/mev_inspect/schemas/miner_payments.py b/mev_inspect/schemas/miner_payments.py index 6f208b3..aa12d59 100644 --- a/mev_inspect/schemas/miner_payments.py +++ b/mev_inspect/schemas/miner_payments.py @@ -1,3 +1,5 @@ +from typing import Optional + from pydantic import BaseModel @@ -11,3 +13,5 @@ class MinerPayment(BaseModel): gas_price: int gas_price_with_coinbase_transfer: int gas_used: int + transaction_to_address: Optional[str] + transaction_from_address: Optional[str]