Use a dedicated table for the last block written. Write new blocks as they come

mev_inspect/block.py

@@ -12,7 +12,7 @@ cache_directory = "./cache"
 
 
 def get_latest_block_number(w3: Web3) -> int:
-    return w3.eth.get_block("latest")["number"]
+    return int(w3.eth.get_block("latest")["number"])
 
 
 def create_from_block_number(

mev_inspect/crud/miner_payments.py

@@ -1,7 +1,5 @@
 import json
-from typing import List, Optional
+from typing import List
-
-from sqlalchemy.sql.expression import func
 
 from mev_inspect.models.miner_payments import MinerPaymentModel
 from mev_inspect.schemas.miner_payments import MinerPayment
@@ -31,11 +29,3 @@ def write_miner_payments(
 
     db_session.bulk_save_objects(models)
     db_session.commit()
-
-
-def get_max_miner_payment_block(db_session) -> Optional[int]:
-    results = db_session.query(func.max(MinerPaymentModel.block_number)).one_or_none()
-    if results is None:
-        return None
-    else:
-        return int(results[0])

run.py

@@ -6,8 +6,12 @@ import time
 from web3 import Web3
 
 from mev_inspect.block import get_latest_block_number
-from mev_inspect.crud.miner_payments import get_max_miner_payment_block
+from mev_inspect.crud.latest_block_update import (
+    find_latest_block_update,
+    update_latest_block,
+)
 from mev_inspect.db import get_session
+from mev_inspect.inspect_block import inspect_block
 from mev_inspect.provider import get_base_provider
 
 
@@ -48,13 +52,34 @@ def run():
     base_provider = get_base_provider(rpc)
     w3 = Web3(base_provider)
 
-    while not killer.kill_now:
+    latest_block_number = get_latest_block_number(w3)
-        latest_block_number = get_latest_block_number(w3)
-        last_written_block = get_max_miner_payment_block(db_session)
 
+    while not killer.kill_now:
+        last_written_block = find_latest_block_update(db_session)
         logger.info(f"Latest block: {latest_block_number}")
         logger.info(f"Last written block: {last_written_block}")
-        time.sleep(5)
+
+        if last_written_block is None or last_written_block < latest_block_number:
+            block_number = (
+                latest_block_number
+                if last_written_block is None
+                else last_written_block + 1
+            )
+
+            logger.info(f"Writing block: {block_number}")
+
+            inspect_block(
+                db_session,
+                base_provider,
+                w3,
+                block_number,
+                should_write_classified_traces=False,
+                should_cache=False,
+            )
+            update_latest_block(db_session, block_number)
+        else:
+            latest_block_number = get_latest_block_number(w3)
+            time.sleep(5)
 
     logger.info("Stopping...")