Merge pull request #60 from flashbots/miner-db

Write miner payments to the DB

alembic/versions/083978d6e455_create_miner_payments_table.py

@@ -0,0 +1,36 @@
+"""Create miner_payments table
+
+Revision ID: 083978d6e455
+Revises: 92f28a2b4f52
+Create Date: 2021-08-30 17:42:25.548130
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "083978d6e455"
+down_revision = "92f28a2b4f52"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "miner_payments",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("transaction_hash", sa.String(66), primary_key=True),
+        sa.Column("transaction_index", sa.Numeric, nullable=False),
+        sa.Column("miner_address", sa.String(256), nullable=False),
+        sa.Column("coinbase_transfer", sa.Numeric, nullable=False),
+        sa.Column("base_fee_per_gas", sa.Numeric, nullable=False),
+        sa.Column("gas_price", sa.Numeric, nullable=False),
+        sa.Column("gas_price_with_coinbase_transfer", sa.Numeric, nullable=False),
+        sa.Column("gas_used", sa.Numeric, nullable=False),
+    )
+
+
+def downgrade():
+    op.drop_table("miner_payments")

alembic/versions/d70c08b4db6f_add_to_address_and_from_address_to_.py

@@ -0,0 +1,32 @@
+"""Add to_address and from_address to miner_payments table
+
+Revision ID: d70c08b4db6f
+Revises: 083978d6e455
+Create Date: 2021-08-30 22:10:04.186251
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "d70c08b4db6f"
+down_revision = "083978d6e455"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "miner_payments",
+        sa.Column("transaction_to_address", sa.String(256), nullable=True),
+    )
+    op.add_column(
+        "miner_payments",
+        sa.Column("transaction_from_address", sa.String(256), nullable=True),
+    )
+
+
+def downgrade():
+    op.drop_column("miner_payments", "transaction_to_address")
+    op.drop_column("miner_payments", "transaction_from_address")

mev_inspect/crud/miner_payments.py

@@ -0,0 +1,31 @@
+import json
+from typing import List
+
+from mev_inspect.models.miner_payments import MinerPaymentModel
+from mev_inspect.schemas.miner_payments import MinerPayment
+
+
+def delete_miner_payments_for_block(
+    db_session,
+    block_number: int,
+) -> None:
+    (
+        db_session.query(MinerPaymentModel)
+        .filter(MinerPaymentModel.block_number == block_number)
+        .delete()
+    )
+
+    db_session.commit()
+
+
+def write_miner_payments(
+    db_session,
+    miner_payments: List[MinerPayment],
+) -> None:
+    models = [
+        MinerPaymentModel(**json.loads(miner_payment.json()))
+        for miner_payment in miner_payments
+    ]
+
+    db_session.bulk_save_objects(models)
+    db_session.commit()

mev_inspect/miner_payments.py

@@ -22,6 +22,12 @@ def get_miner_payments(
 
     for receipt in receipts:
         transaciton_traces = traces_by_transaction_hash[receipt.transaction_hash]
+
+        if len(transaciton_traces) == 0:
+            continue
+
+        first_trace = sorted(transaciton_traces, key=lambda t: t.trace_address)[0]
+
         eth_transfers = get_eth_transfers(transaciton_traces)
         miner_eth_transfers = filter_transfers(
             eth_transfers, to_address=miner_address.lower()
@@ -46,6 +52,8 @@ def get_miner_payments(
                 base_fee_per_gas=base_fee_per_gas,
                 gas_used=receipt.gas_used,
                 coinbase_transfer=coinbase_transfer,
+                transaction_to_address=first_trace.to_address,
+                transaction_from_address=first_trace.from_address,
             )
         )
 

mev_inspect/models/miner_payments.py

@@ -0,0 +1,19 @@
+from sqlalchemy import Column, Numeric, String
+
+from .base import Base
+
+
+class MinerPaymentModel(Base):
+    __tablename__ = "miner_payments"
+
+    block_number = Column(Numeric, nullable=False)
+    transaction_hash = Column(String, primary_key=True)
+    transaction_index = Column(Numeric, nullable=False)
+    miner_address = Column(String, nullable=False)
+    coinbase_transfer = Column(Numeric, nullable=False)
+    base_fee_per_gas = Column(Numeric, nullable=False)
+    gas_price = Column(Numeric, nullable=False)
+    gas_price_with_coinbase_transfer = Column(Numeric, nullable=False)
+    gas_used = Column(Numeric, nullable=False)
+    transaction_from_address = Column(String, nullable=True)
+    transaction_to_address = Column(String, nullable=True)

mev_inspect/schemas/miner_payments.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from pydantic import BaseModel
 
 
@@ -11,3 +13,5 @@ class MinerPayment(BaseModel):
     gas_price: int
     gas_price_with_coinbase_transfer: int
     gas_used: int
+    transaction_to_address: Optional[str]
+    transaction_from_address: Optional[str]

scripts/inspect_block.py

@@ -3,7 +3,10 @@ import json
 import click
 from web3 import Web3
 
+from mev_inspect.arbitrages import get_arbitrages
 from mev_inspect.block import create_from_block_number
+from mev_inspect.classifiers.specs import ALL_CLASSIFIER_SPECS
+from mev_inspect.classifiers.trace import TraceClassifier
 from mev_inspect.crud.arbitrages import (
     delete_arbitrages_for_block,
     write_arbitrages,
@@ -12,10 +15,10 @@ from mev_inspect.crud.classified_traces import (
     delete_classified_traces_for_block,
     write_classified_traces,
 )
-
+from mev_inspect.crud.miner_payments import (
-from mev_inspect.arbitrages import get_arbitrages
+    delete_miner_payments_for_block,
-from mev_inspect.classifiers.specs import ALL_CLASSIFIER_SPECS
+    write_miner_payments,
-from mev_inspect.classifiers.trace import TraceClassifier
+)
 from mev_inspect.crud.swaps import delete_swaps_for_block, write_swaps
 from mev_inspect.db import get_session
 from mev_inspect.miner_payments import get_miner_payments
@@ -78,10 +81,11 @@ def _inspect_block(
     block_number: int,
     should_cache: bool,
     should_print_stats: bool = True,
+    should_print_miner_payments: bool = True,
     should_write_classified_traces: bool = True,
     should_write_swaps: bool = True,
     should_write_arbitrages: bool = True,
-    should_print_miner_payments: bool = True,
+    should_write_miner_payments: bool = True,
 ):
     block = create_from_block_number(base_provider, w3, block_number, should_cache)
 
@@ -127,6 +131,10 @@ def _inspect_block(
     if should_print_miner_payments:
         click.echo(json.dumps([p.dict() for p in miner_payments], indent=4))
 
+    if should_write_miner_payments:
+        delete_miner_payments_for_block(db_session, block_number)
+        write_miner_payments(db_session, miner_payments)
+
 
 def get_stats(classified_traces) -> dict:
     stats: dict = {}