Use a dedicated table for the last block written. Write new blocks as they come

This commit is contained in:
Luke Van Seters 2021-09-13 21:55:19 -04:00
parent 50d04a0b42
commit a9cbe106ad
3 changed files with 32 additions and 17 deletions

View File

@ -12,7 +12,7 @@ cache_directory = "./cache"
def get_latest_block_number(w3: Web3) -> int: def get_latest_block_number(w3: Web3) -> int:
return w3.eth.get_block("latest")["number"] return int(w3.eth.get_block("latest")["number"])
def create_from_block_number( def create_from_block_number(

View File

@ -1,7 +1,5 @@
import json import json
from typing import List, Optional from typing import List
from sqlalchemy.sql.expression import func
from mev_inspect.models.miner_payments import MinerPaymentModel from mev_inspect.models.miner_payments import MinerPaymentModel
from mev_inspect.schemas.miner_payments import MinerPayment from mev_inspect.schemas.miner_payments import MinerPayment
@ -31,11 +29,3 @@ def write_miner_payments(
db_session.bulk_save_objects(models) db_session.bulk_save_objects(models)
db_session.commit() db_session.commit()
def get_max_miner_payment_block(db_session) -> Optional[int]:
results = db_session.query(func.max(MinerPaymentModel.block_number)).one_or_none()
if results is None:
return None
else:
return int(results[0])

31
run.py
View File

@ -6,8 +6,12 @@ import time
from web3 import Web3 from web3 import Web3
from mev_inspect.block import get_latest_block_number from mev_inspect.block import get_latest_block_number
from mev_inspect.crud.miner_payments import get_max_miner_payment_block from mev_inspect.crud.latest_block_update import (
find_latest_block_update,
update_latest_block,
)
from mev_inspect.db import get_session from mev_inspect.db import get_session
from mev_inspect.inspect_block import inspect_block
from mev_inspect.provider import get_base_provider from mev_inspect.provider import get_base_provider
@ -48,12 +52,33 @@ def run():
base_provider = get_base_provider(rpc) base_provider = get_base_provider(rpc)
w3 = Web3(base_provider) w3 = Web3(base_provider)
while not killer.kill_now:
latest_block_number = get_latest_block_number(w3) latest_block_number = get_latest_block_number(w3)
last_written_block = get_max_miner_payment_block(db_session)
while not killer.kill_now:
last_written_block = find_latest_block_update(db_session)
logger.info(f"Latest block: {latest_block_number}") logger.info(f"Latest block: {latest_block_number}")
logger.info(f"Last written block: {last_written_block}") logger.info(f"Last written block: {last_written_block}")
if last_written_block is None or last_written_block < latest_block_number:
block_number = (
latest_block_number
if last_written_block is None
else last_written_block + 1
)
logger.info(f"Writing block: {block_number}")
inspect_block(
db_session,
base_provider,
w3,
block_number,
should_write_classified_traces=False,
should_cache=False,
)
update_latest_block(db_session, block_number)
else:
latest_block_number = get_latest_block_number(w3)
time.sleep(5) time.sleep(5)
logger.info("Stopping...") logger.info("Stopping...")