From 4993bbc8e0b7983622a643315e1e037b5c995cde Mon Sep 17 00:00:00 2001
From: Luke Van Seters <lukevanseters@gmail.com>
Date: Mon, 13 Sep 2021 18:34:40 -0400
Subject: [PATCH 01/11] Create cache dir if not exists

---
 mev_inspect/block.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mev_inspect/block.py b/mev_inspect/block.py
index d1dbee7..dca522d 100644
--- a/mev_inspect/block.py
+++ b/mev_inspect/block.py
@@ -70,6 +70,8 @@ def get_transaction_hashes(calls: List[Trace]) -> List[str]:
 def cache_block(cache_path: Path, block: Block):
     write_mode = "w" if cache_path.is_file() else "x"
 
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+
     with open(cache_path, mode=write_mode) as cache_file:
         cache_file.write(block.json())
 

From e365a2c0c0c9e654431b6eae84a3ded54541d240 Mon Sep 17 00:00:00 2001
From: Luke Van Seters <lukevanseters@gmail.com>
Date: Mon, 13 Sep 2021 18:43:14 -0400
Subject: [PATCH 02/11] Move inspect block logic into mev_inspect module from
 script

---
 pyproject.toml           |   4 +-
 scripts/inspect_block.py | 172 ---------------------------------------
 2 files changed, 2 insertions(+), 174 deletions(-)
 delete mode 100644 scripts/inspect_block.py

diff --git a/pyproject.toml b/pyproject.toml
index 3d7b66d..47b50a3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,8 +41,8 @@ attach = 'scripts.poetry.docker:attach'
 exec = 'scripts.poetry.docker:exec'
 inspect = 'scripts.poetry.inspect:inspect'
 inspect-many = 'scripts.poetry.inspect:inspect_many'
-inspect-block = 'scripts.inspect_block:inspect_block'
-inspect-many-blocks = 'scripts.inspect_block:inspect_many_blocks'
+inspect-block = 'scripts.inspect_commands:inspect_block_command'
+inspect-many-blocks = 'scripts.inspect_commands:inspect_many_blocks_command'
 
 [tool.black]
 exclude = '''
diff --git a/scripts/inspect_block.py b/scripts/inspect_block.py
deleted file mode 100644
index 7a72bfa..0000000
--- a/scripts/inspect_block.py
+++ /dev/null
@@ -1,172 +0,0 @@
-import json
-
-import click
-from web3 import Web3
-
-from mev_inspect.arbitrages import get_arbitrages
-from mev_inspect.block import create_from_block_number
-from mev_inspect.classifiers.trace import TraceClassifier
-from mev_inspect.crud.arbitrages import (
-    delete_arbitrages_for_block,
-    write_arbitrages,
-)
-from mev_inspect.crud.classified_traces import (
-    delete_classified_traces_for_block,
-    write_classified_traces,
-)
-from mev_inspect.crud.miner_payments import (
-    delete_miner_payments_for_block,
-    write_miner_payments,
-)
-from mev_inspect.crud.swaps import delete_swaps_for_block, write_swaps
-from mev_inspect.db import get_session
-from mev_inspect.miner_payments import get_miner_payments
-from mev_inspect.swaps import get_swaps
-from mev_inspect.retry import http_retry_with_backoff_request_middleware
-
-
-@click.group()
-def cli():
-    pass
-
-
-@cli.command()
-@click.argument("block_number", type=int)
-@click.argument("rpc")
-@click.option("--cache/--no-cache", default=True)
-def inspect_block(block_number: int, rpc: str, cache: bool):
-    base_provider = _get_base_provider(rpc)
-    w3 = Web3(base_provider)
-
-    if not cache:
-        click.echo("Skipping cache")
-
-    _inspect_block(base_provider, w3, block_number, should_cache=cache)
-
-
-@cli.command()
-@click.argument("after_block", type=int)
-@click.argument("before_block", type=int)
-@click.argument("rpc")
-@click.option("--cache/--no-cache", default=True)
-def inspect_many_blocks(after_block: int, before_block: int, rpc: str, cache: bool):
-    base_provider = _get_base_provider(rpc)
-    w3 = Web3(base_provider)
-
-    if not cache:
-        click.echo("Skipping cache")
-
-    for i, block_number in enumerate(range(after_block, before_block)):
-        block_message = (
-            f"Running for {block_number} ({i+1}/{before_block - after_block})"
-        )
-        dashes = "-" * len(block_message)
-        click.echo(dashes)
-        click.echo(block_message)
-        click.echo(dashes)
-
-        _inspect_block(
-            base_provider,
-            w3,
-            block_number,
-            should_print_stats=False,
-            should_write_classified_traces=False,
-            should_cache=cache,
-        )
-
-
-def _inspect_block(
-    base_provider,
-    w3: Web3,
-    block_number: int,
-    should_cache: bool,
-    should_print_stats: bool = True,
-    should_print_miner_payments: bool = True,
-    should_write_classified_traces: bool = True,
-    should_write_swaps: bool = True,
-    should_write_arbitrages: bool = True,
-    should_write_miner_payments: bool = True,
-):
-    block = create_from_block_number(base_provider, w3, block_number, should_cache)
-
-    click.echo(f"Total traces: {len(block.traces)}")
-
-    total_transactions = len(
-        set(t.transaction_hash for t in block.traces if t.transaction_hash is not None)
-    )
-    click.echo(f"Total transactions: {total_transactions}")
-
-    trace_clasifier = TraceClassifier()
-    classified_traces = trace_clasifier.classify(block.traces)
-    click.echo(f"Returned {len(classified_traces)} classified traces")
-
-    db_session = get_session()
-
-    if should_write_classified_traces:
-        delete_classified_traces_for_block(db_session, block_number)
-        write_classified_traces(db_session, classified_traces)
-
-    swaps = get_swaps(classified_traces)
-    click.echo(f"Found {len(swaps)} swaps")
-
-    if should_write_swaps:
-        delete_swaps_for_block(db_session, block_number)
-        write_swaps(db_session, swaps)
-
-    arbitrages = get_arbitrages(swaps)
-    click.echo(f"Found {len(arbitrages)} arbitrages")
-
-    if should_write_arbitrages:
-        delete_arbitrages_for_block(db_session, block_number)
-        write_arbitrages(db_session, arbitrages)
-
-    if should_print_stats:
-        stats = get_stats(classified_traces)
-        click.echo(json.dumps(stats, indent=4))
-
-    miner_payments = get_miner_payments(
-        block.miner, block.base_fee_per_gas, classified_traces, block.receipts
-    )
-
-    if should_print_miner_payments:
-        click.echo(json.dumps([p.dict() for p in miner_payments], indent=4))
-
-    if should_write_miner_payments:
-        delete_miner_payments_for_block(db_session, block_number)
-        write_miner_payments(db_session, miner_payments)
-
-
-def get_stats(classified_traces) -> dict:
-    stats: dict = {}
-
-    for trace in classified_traces:
-        protocol = str(trace.protocol)
-        abi_name = trace.abi_name
-        classification = trace.classification.value
-        signature = trace.function_signature
-
-        protocol_stats = stats.get(protocol, {})
-        abi_name_stats = protocol_stats.get(abi_name, {})
-        class_stats = abi_name_stats.get(classification, {})
-        signature_count = class_stats.get(signature, 0)
-        class_stats[signature] = signature_count + 1
-        abi_name_stats[classification] = class_stats
-        protocol_stats[abi_name] = abi_name_stats
-        stats[protocol] = protocol_stats
-
-    return stats
-
-
-def _get_base_provider(rpc: str) -> Web3.HTTPProvider:
-    base_provider = Web3.HTTPProvider(rpc)
-    base_provider.middlewares.remove("http_retry_request")
-    base_provider.middlewares.add(
-        http_retry_with_backoff_request_middleware,
-        "http_retry_with_backoff",
-    )
-
-    return base_provider
-
-
-if __name__ == "__main__":
-    cli()

From 768de19b6035bf85879a006d67385312f450ed99 Mon Sep 17 00:00:00 2001
From: Luke Van Seters <lukevanseters@gmail.com>
Date: Mon, 13 Sep 2021 18:44:04 -0400
Subject: [PATCH 03/11] Move inspect block logic into mev_inspect module from
 script

---
 mev_inspect/inspect_block.py | 73 ++++++++++++++++++++++++++++++++++++
 scripts/inspect_commands.py  | 71 +++++++++++++++++++++++++++++++++++
 2 files changed, 144 insertions(+)
 create mode 100644 mev_inspect/inspect_block.py
 create mode 100644 scripts/inspect_commands.py

diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py
new file mode 100644
index 0000000..3b19345
--- /dev/null
+++ b/mev_inspect/inspect_block.py
@@ -0,0 +1,73 @@
+from web3 import Web3
+
+from mev_inspect.arbitrages import get_arbitrages
+from mev_inspect.block import create_from_block_number
+from mev_inspect.classifiers.trace import TraceClassifier
+from mev_inspect.crud.arbitrages import (
+    delete_arbitrages_for_block,
+    write_arbitrages,
+)
+from mev_inspect.crud.classified_traces import (
+    delete_classified_traces_for_block,
+    write_classified_traces,
+)
+from mev_inspect.crud.miner_payments import (
+    delete_miner_payments_for_block,
+    write_miner_payments,
+)
+from mev_inspect.crud.swaps import delete_swaps_for_block, write_swaps
+from mev_inspect.db import get_session
+from mev_inspect.miner_payments import get_miner_payments
+from mev_inspect.swaps import get_swaps
+
+
+def inspect_block(
+    base_provider,
+    w3: Web3,
+    block_number: int,
+    should_cache: bool,
+    should_write_classified_traces: bool = True,
+    should_write_swaps: bool = True,
+    should_write_arbitrages: bool = True,
+    should_write_miner_payments: bool = True,
+):
+    block = create_from_block_number(base_provider, w3, block_number, should_cache)
+
+    print(f"Total traces: {len(block.traces)}")
+
+    total_transactions = len(
+        set(t.transaction_hash for t in block.traces if t.transaction_hash is not None)
+    )
+    print(f"Total transactions: {total_transactions}")
+
+    trace_clasifier = TraceClassifier()
+    classified_traces = trace_clasifier.classify(block.traces)
+    print(f"Returned {len(classified_traces)} classified traces")
+
+    db_session = get_session()
+
+    if should_write_classified_traces:
+        delete_classified_traces_for_block(db_session, block_number)
+        write_classified_traces(db_session, classified_traces)
+
+    swaps = get_swaps(classified_traces)
+    print(f"Found {len(swaps)} swaps")
+
+    if should_write_swaps:
+        delete_swaps_for_block(db_session, block_number)
+        write_swaps(db_session, swaps)
+
+    arbitrages = get_arbitrages(swaps)
+    print(f"Found {len(arbitrages)} arbitrages")
+
+    if should_write_arbitrages:
+        delete_arbitrages_for_block(db_session, block_number)
+        write_arbitrages(db_session, arbitrages)
+
+    miner_payments = get_miner_payments(
+        block.miner, block.base_fee_per_gas, classified_traces, block.receipts
+    )
+
+    if should_write_miner_payments:
+        delete_miner_payments_for_block(db_session, block_number)
+        write_miner_payments(db_session, miner_payments)
diff --git a/scripts/inspect_commands.py b/scripts/inspect_commands.py
new file mode 100644
index 0000000..312dedc
--- /dev/null
+++ b/scripts/inspect_commands.py
@@ -0,0 +1,71 @@
+import click
+from web3 import Web3
+
+from mev_inspect.inspect_block import inspect_block
+from mev_inspect.retry import http_retry_with_backoff_request_middleware
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+@click.argument("block_number", type=int)
+@click.argument("rpc")
+@click.option("--cache/--no-cache", default=True)
+def inspect_block_command(block_number: int, rpc: str, cache: bool):
+    base_provider = _get_base_provider(rpc)
+    w3 = Web3(base_provider)
+
+    if not cache:
+        click.echo("Skipping cache")
+
+    inspect_block(base_provider, w3, block_number, should_cache=cache)
+
+
+@cli.command()
+@click.argument("after_block", type=int)
+@click.argument("before_block", type=int)
+@click.argument("rpc")
+@click.option("--cache/--no-cache", default=True)
+def inspect_many_blocks_command(
+    after_block: int, before_block: int, rpc: str, cache: bool
+):
+    base_provider = _get_base_provider(rpc)
+    w3 = Web3(base_provider)
+
+    if not cache:
+        click.echo("Skipping cache")
+
+    for i, block_number in enumerate(range(after_block, before_block)):
+        block_message = (
+            f"Running for {block_number} ({i+1}/{before_block - after_block})"
+        )
+        dashes = "-" * len(block_message)
+        click.echo(dashes)
+        click.echo(block_message)
+        click.echo(dashes)
+
+        inspect_block(
+            base_provider,
+            w3,
+            block_number,
+            should_write_classified_traces=False,
+            should_cache=cache,
+        )
+
+
+def _get_base_provider(rpc: str) -> Web3.HTTPProvider:
+    base_provider = Web3.HTTPProvider(rpc)
+    base_provider.middlewares.remove("http_retry_request")
+    base_provider.middlewares.add(
+        http_retry_with_backoff_request_middleware,
+        "http_retry_with_backoff",
+    )
+
+    return base_provider
+
+
+if __name__ == "__main__":
+    cli()

From 66c22682e8eb9d8dbb5ddb2ef55ee35cdb77463e Mon Sep 17 00:00:00 2001
From: Luke Van Seters <lukevanseters@gmail.com>
Date: Mon, 13 Sep 2021 18:44:35 -0400
Subject: [PATCH 04/11] Get empty list default for miner payments

---
 mev_inspect/miner_payments.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/mev_inspect/miner_payments.py b/mev_inspect/miner_payments.py
index cb3d80f..8e74729 100644
--- a/mev_inspect/miner_payments.py
+++ b/mev_inspect/miner_payments.py
@@ -21,7 +21,9 @@ def get_miner_payments(
     traces_by_transaction_hash = get_traces_by_transaction_hash(traces)
 
     for receipt in receipts:
-        transaciton_traces = traces_by_transaction_hash[receipt.transaction_hash]
+        transaciton_traces = traces_by_transaction_hash.get(
+            receipt.transaction_hash, []
+        )
 
         if len(transaciton_traces) == 0:
             continue

From e92c36d30ac304bab1577a661846e90a15bc7a07 Mon Sep 17 00:00:00 2001
From: Luke Van Seters <lukevanseters@gmail.com>
Date: Mon, 13 Sep 2021 19:57:31 -0400
Subject: [PATCH 05/11] Move DB session out

---
 mev_inspect/inspect_block.py | 1 +
 scripts/inspect_commands.py  | 7 ++++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py
index 3b19345..4445869 100644
--- a/mev_inspect/inspect_block.py
+++ b/mev_inspect/inspect_block.py
@@ -22,6 +22,7 @@ from mev_inspect.swaps import get_swaps
 
 
 def inspect_block(
+    db_session,
     base_provider,
     w3: Web3,
     block_number: int,
diff --git a/scripts/inspect_commands.py b/scripts/inspect_commands.py
index 312dedc..ec38408 100644
--- a/scripts/inspect_commands.py
+++ b/scripts/inspect_commands.py
@@ -1,6 +1,7 @@
 import click
 from web3 import Web3
 
+from mev_inspect.db import get_session
 from mev_inspect.inspect_block import inspect_block
 from mev_inspect.retry import http_retry_with_backoff_request_middleware
 
@@ -15,13 +16,14 @@ def cli():
 @click.argument("rpc")
 @click.option("--cache/--no-cache", default=True)
 def inspect_block_command(block_number: int, rpc: str, cache: bool):
+    db_session = get_session()
     base_provider = _get_base_provider(rpc)
     w3 = Web3(base_provider)
 
     if not cache:
         click.echo("Skipping cache")
 
-    inspect_block(base_provider, w3, block_number, should_cache=cache)
+    inspect_block(db_session, base_provider, w3, block_number, should_cache=cache)
 
 
 @cli.command()
@@ -32,6 +34,8 @@ def inspect_block_command(block_number: int, rpc: str, cache: bool):
 def inspect_many_blocks_command(
     after_block: int, before_block: int, rpc: str, cache: bool
 ):
+
+    db_session = get_session()
     base_provider = _get_base_provider(rpc)
     w3 = Web3(base_provider)
 
@@ -48,6 +52,7 @@ def inspect_many_blocks_command(
         click.echo(dashes)
 
         inspect_block(
+            db_session,
             base_provider,
             w3,
             block_number,

From 7a53816d740136c721eea3473d3f603ee75bb428 Mon Sep 17 00:00:00 2001
From: Luke Van Seters <lukevanseters@gmail.com>
Date: Mon, 13 Sep 2021 20:33:00 -0400
Subject: [PATCH 06/11] Add auto-restart of process. Add GracefulKiller

---
 Dockerfile                                    |   2 +-
 Tiltfile                                      |   9 +-
 k8s/app.yaml                                  |   2 +-
 mev_inspect/inspect_block.py                  |   3 -
 run.py                                        |  36 +++-
 run.sh                                        |   3 +
 tilt_modules/extensions.json                  |   5 +
 tilt_modules/restart_process/.gitignore       |   1 +
 tilt_modules/restart_process/Dockerfile       |  15 ++
 tilt_modules/restart_process/README.md        | 181 ++++++++++++++++++
 tilt_modules/restart_process/Tiltfile         | 146 ++++++++++++++
 tilt_modules/restart_process/release.sh       |  24 +++
 tilt_modules/restart_process/test/Dockerfile  |   5 +
 tilt_modules/restart_process/test/Tiltfile    |   5 +
 .../restart_process/test/custom.Tiltfile      |   9 +
 tilt_modules/restart_process/test/fail.sh     |   9 +
 tilt_modules/restart_process/test/job.yaml    |  12 ++
 .../restart_process/test/test-custom.sh       |  21 ++
 .../restart_process/test/test-docker.sh       |  29 +++
 tilt_modules/restart_process/test/test.sh     |   7 +
 .../restart_process/tilt-restart-wrapper.go   |  74 +++++++
 21 files changed, 588 insertions(+), 10 deletions(-)
 create mode 100755 run.sh
 create mode 100644 tilt_modules/restart_process/.gitignore
 create mode 100644 tilt_modules/restart_process/Dockerfile
 create mode 100644 tilt_modules/restart_process/README.md
 create mode 100644 tilt_modules/restart_process/Tiltfile
 create mode 100755 tilt_modules/restart_process/release.sh
 create mode 100644 tilt_modules/restart_process/test/Dockerfile
 create mode 100644 tilt_modules/restart_process/test/Tiltfile
 create mode 100644 tilt_modules/restart_process/test/custom.Tiltfile
 create mode 100755 tilt_modules/restart_process/test/fail.sh
 create mode 100644 tilt_modules/restart_process/test/job.yaml
 create mode 100755 tilt_modules/restart_process/test/test-custom.sh
 create mode 100755 tilt_modules/restart_process/test/test-docker.sh
 create mode 100755 tilt_modules/restart_process/test/test.sh
 create mode 100644 tilt_modules/restart_process/tilt-restart-wrapper.go

diff --git a/Dockerfile b/Dockerfile
index 5f6cc69..f2323eb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -18,4 +18,4 @@ COPY . /app
 # easter eggs 😝
 RUN echo "PS1='🕵️:\[\033[1;36m\]\h \[\033[1;34m\]\W\[\033[0;35m\]\[\033[1;36m\]$ \[\033[0m\]'" >> ~/.bashrc
 
-CMD [ "python", "run.py"]
+ENTRYPOINT [ "/app/run.sh"]
diff --git a/Tiltfile b/Tiltfile
index 25e1948..abc17cb 100644
--- a/Tiltfile
+++ b/Tiltfile
@@ -1,22 +1,25 @@
 load('ext://helm_remote', 'helm_remote')
+load('ext://restart_process', 'docker_build_with_restart')
+load('ext://secret', 'secret_from_dict')
+
 helm_remote("postgresql",
             repo_name='bitnami',
             repo_url='https://charts.bitnami.com/bitnami',
             set=["postgresqlPassword=password", "postgresqlDatabase=mev_inspect"],
 )
 
-load('ext://secret', 'secret_from_dict')
 k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = {
     "username" : "postgres",
     "password": "password",
 }))
 
-docker_build('mev-inspect-py', '.',
+docker_build_with_restart('mev-inspect-py', '.',
+    entrypoint="/app/run.sh",
     live_update=[
         sync('.', '/app'),
         run('cd /app && poetry install',
             trigger='./pyproject.toml'),
     ],
+    platform='linux/arm64',
 )
-
 k8s_yaml("k8s/app.yaml")
diff --git a/k8s/app.yaml b/k8s/app.yaml
index 46e0946..da45f2e 100644
--- a/k8s/app.yaml
+++ b/k8s/app.yaml
@@ -17,7 +17,7 @@ spec:
       containers:
       - name: mev-inspect
         image: mev-inspect-py
-        command: [ "python", "run.py"]
+        command: [ "/app/run.sh" ]
         env:
         - name: POSTGRES_USER
           valueFrom:
diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py
index 4445869..c6e04af 100644
--- a/mev_inspect/inspect_block.py
+++ b/mev_inspect/inspect_block.py
@@ -16,7 +16,6 @@ from mev_inspect.crud.miner_payments import (
     write_miner_payments,
 )
 from mev_inspect.crud.swaps import delete_swaps_for_block, write_swaps
-from mev_inspect.db import get_session
 from mev_inspect.miner_payments import get_miner_payments
 from mev_inspect.swaps import get_swaps
 
@@ -45,8 +44,6 @@ def inspect_block(
     classified_traces = trace_clasifier.classify(block.traces)
     print(f"Returned {len(classified_traces)} classified traces")
 
-    db_session = get_session()
-
     if should_write_classified_traces:
         delete_classified_traces_for_block(db_session, block_number)
         write_classified_traces(db_session, classified_traces)
diff --git a/run.py b/run.py
index 30dc5d1..22b542a 100644
--- a/run.py
+++ b/run.py
@@ -1,5 +1,37 @@
+import logging
+import signal
 import time
 
+logging.basicConfig(filename="app.log", level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+
+class GracefulKiller:
+    """
+    handle sigint / sigterm gracefully
+    taken from https://stackoverflow.com/a/31464349
+    """
+
+    signal_names = {signal.SIGINT: "SIGINT", signal.SIGTERM: "SIGTERM"}
+
+    def __init__(self):
+        self.kill_now = False
+        signal.signal(signal.SIGINT, self.exit_gracefully)
+        signal.signal(signal.SIGTERM, self.exit_gracefully)
+
+    def exit_gracefully(self, signum, frame):  # pylint: disable=unused-argument
+        signal_name = self.signal_names[signum]
+        print(f"Received {signal_name} signal")
+        print("Cleaning up resources. End of process")
+        self.kill_now = True
+
+
 if __name__ == "__main__":
-    while True:
-        time.sleep(30)
+    logger.info("Starting...")
+    killer = GracefulKiller()
+
+    while not killer.kill_now:
+        logger.info("Running...")
+        time.sleep(5)
+
+    logger.info("Stopping...")
diff --git a/run.sh b/run.sh
new file mode 100755
index 0000000..839b387
--- /dev/null
+++ b/run.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python run.py
diff --git a/tilt_modules/extensions.json b/tilt_modules/extensions.json
index c59980c..4d58ca7 100644
--- a/tilt_modules/extensions.json
+++ b/tilt_modules/extensions.json
@@ -14,6 +14,11 @@
       "Name": "secret",
       "ExtensionRegistry": "https://github.com/tilt-dev/tilt-extensions",
       "TimeFetched": "2021-09-09T08:57:26.199313-06:00"
+    },
+    {
+      "Name": "restart_process",
+      "ExtensionRegistry": "https://github.com/tilt-dev/tilt-extensions",
+      "TimeFetched": "2021-09-13T20:14:11.011803-04:00"
     }
   ]
 }
\ No newline at end of file
diff --git a/tilt_modules/restart_process/.gitignore b/tilt_modules/restart_process/.gitignore
new file mode 100644
index 0000000..3758a00
--- /dev/null
+++ b/tilt_modules/restart_process/.gitignore
@@ -0,0 +1 @@
+tilt-restart-wrapper
diff --git a/tilt_modules/restart_process/Dockerfile b/tilt_modules/restart_process/Dockerfile
new file mode 100644
index 0000000..2ad4b08
--- /dev/null
+++ b/tilt_modules/restart_process/Dockerfile
@@ -0,0 +1,15 @@
+FROM alpine/git
+
+RUN apk update && apk add make
+RUN apk add build-base
+
+RUN git clone https://github.com/eradman/entr.git /entr
+WORKDIR /entr
+RUN git checkout c564e6bdca1dfe2177d1224363cad734158863ad
+RUN ./configure; CFLAGS="-static" make install
+
+FROM scratch
+
+COPY --from=0  /usr/local/bin/entr /
+
+ADD tilt-restart-wrapper /
diff --git a/tilt_modules/restart_process/README.md b/tilt_modules/restart_process/README.md
new file mode 100644
index 0000000..b47f82e
--- /dev/null
+++ b/tilt_modules/restart_process/README.md
@@ -0,0 +1,181 @@
+# Restart Process
+
+This extension helps create images that can restart on `live_update`:
+
+- `docker_build_with_restart`: wraps a `docker_build` call
+- `custom_build_with_restart`: wraps a `custom_build` call
+
+At the end of a `live_update`, the container's process will rerun itself.
+
+(Use it in place of the `restart_container()` Live Update step, which has been deprecated for Kubernetes resources.)
+
+## When to Use
+Use this extension when you have an image and you want to re-execute its entrypoint/command as part of a `live_update`.
+
+E.g. if your app is a static binary, you'll probably need to re-execute the binary for any changes you made to take effect.
+
+(If your app has hot reloading capabilities--i.e. it can detect and incorporate changes to its source code without needing to restart--you probably don't need this extension.)
+
+### Unsupported Cases
+This extension does NOT support process restarts for:
+- Images built with `custom_build` using any of the `skips_local_docker`, `disable_push`, or `tag` parameters.
+- Images run in Docker Compose resources (use the [`restart_container()`](https://docs.tilt.dev/api.html#api.restart_container) builtin instead)
+- Images without a shell (e.g. `scratch`, `distroless`)
+- Container commands specified as `command` in Kubernetes YAML will be overridden by this extension.
+  - However, the `args` field is still available; [reach out](https://tilt.dev/contact) if you need help navigating the interplay between Tilt and these YAML values
+- CRDs
+
+If this extension doesn't work for your use case, [see our docs for alternatives](https://docs.tilt.dev/live_update_reference.html#restarting-your-process).
+
+Run into a bug? Need a use case that we don't yet support? Let us know--[open an issue](https://github.com/tilt-dev/tilt-extensions/issues) or [contact us](https://tilt.dev/contact).
+
+## How to Use
+
+Import this extension by putting the following at the top of your Tiltfile:
+```python
+load('ext://restart_process', 'docker_build_with_restart')
+```
+
+For the image that needs the process restart, replace your existing `docker_build` call:
+```python
+docker_build(
+    'foo-image',
+    './foo',
+    arg1=val1,
+    arg2=val2,
+    live_update=[x, y, z...]
+)
+```
+with a `docker_build_with_restart` call:
+```python
+docker_build_with_restart(
+    'foo-image',
+    './foo',
+    entrypoint='/go/bin/foo',
+    arg1=val1,
+    arg2=val2,
+    live_update=[x, y, z...]
+)
+```
+The call above looks just like the initial `docker_build` call except for one added parameter, `entrypoint` (in this example, `/go/bin/foo`). This is the command that you want to run on container start and _re-run_ on Live Update.
+
+A custom_build call looks similar:
+
+```python
+load('ext://restart_process', 'custom_build_with_restart')
+
+custom_build_with_restart(
+    'foo-image',
+    'docker build -t $EXPECTED_REF ./foo',
+    deps=['./foo'],
+    live_update=[sync(...)]
+)
+```
+
+### Troubleshooting
+#### `failed running [touch /tmp/.restart-proc']`
+If you see an error of the form:
+```
+ERROR: Build Failed: ImageBuild: executor failed running [touch /tmp/.restart-proc']: exit code: 1
+```
+this often means that your Dockerfile user ([see docs](https://docs.docker.com/engine/reference/builder/#user)) doesn't have permission to write to the file we use to signal a process restart. Use the `restart_file` parameter to specify a file that your Dockerfile user definitely has write access to.
+
+### API
+```python
+def docker_build_with_restart(ref: str, context: str,
+    entrypoint: Union[str, List[str]],
+    live_update: List[LiveUpdateStep],
+    base_suffix: str = '-base',
+    restart_file: str = '/.restart-proc',
+    trigger: Union[str, List[str]] = [],
+    **kwargs
+):
+    """Args:
+      ref: name for this image (e.g. 'myproj/backend' or 'myregistry/myproj/backend'); as the parameter of the same name in docker_build
+      context: path to use as the Docker build context; as the parameter of the same name in docker_build
+      entrypoint: the command to be (re-)executed when the container starts or when a live_update is run
+      live_update: set of steps for updating a running container; as the parameter of the same name in docker_build
+      base_suffix: suffix for naming the base image, applied as {ref}{base_suffix}
+      restart_file: file that Tilt will update during a live_update to signal the entrypoint to rerun
+      trigger: (optional) list of local paths. If specified, the process will ONLY be restarted when there are changes
+               to the given file(s); as the parameter of the same name in the LiveUpdate `run` step.
+      **kwargs: will be passed to the underlying `docker_build` call
+    """
+    
+
+def custom_build_with_restart(ref: str, command: str, deps: List[str], entrypoint,
+
+    entrypoint: Union[str, List[str]],
+    live_update: List[LiveUpdateStep],
+    base_suffix: str = '-base',
+    restart_file: str = '/.restart-proc',
+    trigger: Union[str, List[str]] = [],
+    , **kwargs
+):
+    """
+     Args:
+      ref: name for this image (e.g. 'myproj/backend' or 'myregistry/myproj/backend'); as the parameter of the same name in custom_build
+      command: build command for building your image
+      deps: source dependencies of the custom build
+      entrypoint: the command to be (re-)executed when the container starts or when a live_update is run
+      live_update: set of steps for updating a running container; as the parameter of the same name in custom_build
+      base_suffix: suffix for naming the base image, applied as {ref}{base_suffix}
+      restart_file: file that Tilt will update during a live_update to signal the entrypoint to rerun
+      trigger: (optional) list of local paths. If specified, the process will ONLY be restarted when there are changes
+               to the given file(s); as the parameter of the same name in the LiveUpdate `run` step.
+      **kwargs: will be passed to the underlying `custom_build` call
+    """
+```
+
+## What's Happening Under the Hood
+*If you're a casual user/just want to get your app running, you can stop reading now. However, if you want to dig deep and know exactly what's going on, or are trying to debug weird behavior, read on.*
+
+This extension wraps commands in `tilt-restart-wrapper`, which makes use of [`entr`](https://github.com/eradman/entr/)
+to run arbitrary commands whenever a specified file changes. Specifically, we override the container's entrypoint with the following:
+
+```
+/tilt-restart-wrapper --watch_file='/.restart-proc' <entrypoint>
+```
+
+This invocation says:
+- when the container starts, run <entrypoint>
+- whenever the `/.restart-proc` file changes, re-execute <entrypoint>
+
+We also set the following as the last `live_update` step:
+```python
+run('date > /.restart-proc')
+```
+
+Because `tilt-restart-wrapper` will re-execute the entrypoint whenever `/.restart-proc'` changes, the above `run` step will cause the entrypoint to re-run.
+
+#### Provide `tilt-restart-wrapper`
+For this all to work, the `entr` binary must be available on the Docker image. The easiest solution would be to call e.g. `apt-get install entr` in the Dockerfile, but different base images will have different package managers; rather than grapple with that, we've made a statically linked binary available on Docker image: [`tiltdev/entr`](https://hub.docker.com/repository/docker/tiltdev/entr).
+
+To build `image-foo`, this extension will:
+- build your image as normal (via `docker_build`, with all of your specified args/kwargs) but with the name `image-foo-base`
+- build `image-foo` (the actual image that will be used in your resource) as a _child_ of `image-foo-base`, with the `tilt-process-wrapper` and its dependencies available
+
+Thus, the final image produced is tagged `image-foo` and has all the properties of your original `docker_build`, plus access to the `tilt-restart-wrapper` binary.
+
+#### Why a Wrapper?
+Why bother with `tilt-restart-wrapper` rather than just calling `entr` directly?
+
+Because in its canonical invocation, `entr` requires that the file(s) to watch be piped via stdin, i.e. it is invoked like:
+```
+echo "/.restart-proc" | entr -rz /bin/my-app
+```
+
+When specified as a `command` in Kubernetes or Docker Compose YAML (this is how Tilt overrides entrypoints), the above would therefore need to be executed as shell:
+```
+/bin/sh -c 'echo "/.restart-proc" | entr -rz /bin/my-app'
+```
+Any `args` specified in Kubernetes/Docker Compose are attached to the end of this call, and therefore in this case would apply TO THE `/bin/sh -c` CALL, rather than to the actual command run by `entr`; that is, any `args` specified by the user would be effectively ignored.
+
+In order to make `entr` usable without a shell, this extension uses [a simple binary](/restart_process/tilt-restart-wrapper.go) that invokes `entr` and writes to its stdin.
+
+Note: ideally `entr` could accept files-to-watch via flag instead of stdin, but (for a number of good reasons) this feature isn't likely to be added any time soon (see [entr#33](https://github.com/eradman/entr/issues/33)).
+
+## For Maintainers: Releasing
+If you have push access to the `tiltdev` repository on DockerHub, you can release a new version of the binaries used by this extension like so:
+1. run `release.sh` (builds `tilt-restart-wrapper` from source, builds and pushes a Docker image with the new binary and a fresh binary of `entr` also installed from source)
+2. update the image tag in the [Tiltfile](/restart_process/Tiltfile) with the tag you just pushed (you'll find the image referenced in the Dockerfile contents of the child image--look for "FROM tiltdev/restart-helper")
diff --git a/tilt_modules/restart_process/Tiltfile b/tilt_modules/restart_process/Tiltfile
new file mode 100644
index 0000000..e732621
--- /dev/null
+++ b/tilt_modules/restart_process/Tiltfile
@@ -0,0 +1,146 @@
+RESTART_FILE = '/tmp/.restart-proc'
+TYPE_RESTART_CONTAINER_STEP = 'live_update_restart_container_step'
+
+KWARGS_BLACKLIST = [
+    # since we'll be passing `dockerfile_contents` when building the
+    # child image, remove any kwargs that might conflict
+    'dockerfile', 'dockerfile_contents',
+
+    # 'target' isn't relevant to our child build--if we pass this arg,
+    # Docker will just fail to find the specified stage and error out
+    'target',
+]
+
+# Arguments to custom_build that don't apply to the docker_build.
+_CUSTOM_BUILD_KWARGS_BLACKLIST = [
+    'tag',
+    'command_bat',
+    'outputs_image_ref_to',
+    'disable_push',
+]
+
+_ext_dir = os.getcwd()
+
+# shared code between the two restart functions
+def _helper(base_ref, ref, entrypoint, live_update, restart_file=RESTART_FILE, trigger=None, exit_policy='restart', **kwargs):
+    if not trigger:
+        trigger = []
+
+    # declare a new docker build that adds a static binary of tilt-restart-wrapper
+    # (which makes use of `entr` to watch files and restart processes) to the user's image
+    df = '''
+    FROM tiltdev/restart-helper:2021-08-09 as restart-helper
+
+    FROM {}
+    RUN ["touch", "{}"]
+    COPY --from=restart-helper /tilt-restart-wrapper /
+    COPY --from=restart-helper /entr /
+    '''.format(base_ref, restart_file)
+
+    # Change the entrypoint to use `tilt-restart-wrapper`.
+    # `tilt-restart-wrapper` makes use of `entr` (https://github.com/eradman/entr/) to
+    # re-execute $entrypoint whenever $restart_file changes
+    entrypoint_with_entr = ["/tilt-restart-wrapper", "--watch_file={}".format(restart_file)]
+    if exit_policy == 'continue':
+        entrypoint_with_entr = entrypoint_with_entr + ["--entr_flags=-r"]
+    if type(entrypoint) == type(""):
+        entrypoint_with_entr = entrypoint_with_entr + ["sh", "-c", entrypoint]
+    elif type(entrypoint) == type([]):
+        entrypoint_with_entr = entrypoint_with_entr + entrypoint
+    else:
+        fail("`entrypoint` must be a string or list of strings: got {}".format(type(entrypoint)))
+
+    # last live_update step should always be to modify $restart_file, which
+    # triggers the process wrapper to rerun $entrypoint
+    # NB: write `date` instead of just `touch`ing because `entr` doesn't respond
+    # to timestamp changes, only writes (see https://github.com/eradman/entr/issues/32)
+    live_update = live_update + [run('date > {}'.format(restart_file), trigger=trigger)]
+
+    # We don't need a real context. See:
+    # https://github.com/tilt-dev/tilt/issues/3897
+    context = _ext_dir
+
+    docker_build(ref, context, entrypoint=entrypoint_with_entr, dockerfile_contents=df,
+                 live_update=live_update, **kwargs)
+
+def docker_build_with_restart(ref, context, entrypoint, live_update,
+                              base_suffix='-tilt_docker_build_with_restart_base', restart_file=RESTART_FILE,
+                              trigger=None, exit_policy='restart', **kwargs):
+    """Wrap a docker_build call and its associated live_update steps so that the last step
+    of any live update is to rerun the given entrypoint.
+
+     Args:
+      ref: name for this image (e.g. 'myproj/backend' or 'myregistry/myproj/backend'); as the parameter of the same name in docker_build
+      context: path to use as the Docker build context; as the parameter of the same name in docker_build
+      entrypoint: the command to be (re-)executed when the container starts or when a live_update is run
+      live_update: set of steps for updating a running container; as the parameter of the same name in docker_build
+      base_suffix: suffix for naming the base image, applied as {ref}{base_suffix}
+      restart_file: file that Tilt will update during a live_update to signal the entrypoint to rerun
+      trigger: (optional) list of local paths. If specified, the process will ONLY be restarted when there are changes
+               to the given file(s); as the parameter of the same name in the LiveUpdate `run` step.
+      **kwargs: will be passed to the underlying `docker_build` call
+    """
+
+    # first, validate the given live_update steps
+    if len(live_update) == 0:
+        fail("`docker_build_with_restart` requires at least one live_update step")
+    for step in live_update:
+        if type(step) == TYPE_RESTART_CONTAINER_STEP:
+            fail("`docker_build_with_restart` is not compatible with live_update step: " +
+                 "`restart_container()` (this extension is meant to REPLACE restart_container() )")
+
+    # rename the original image to make it a base image and declare a docker_build for it
+    base_ref = '{}{}'.format(ref, base_suffix)
+    docker_build(base_ref, context, **kwargs)
+
+    # Clean kwargs for building the child image (which builds on user's specified
+    # image and copies in Tilt's restart wrapper). In practice, this means removing
+    # kwargs that were relevant to building the user's specified image but are NOT
+    # relevant to building the child image / may conflict with args we specifically
+    # pass for the child image.
+    cleaned_kwargs = {k: v for k, v in kwargs.items() if k not in KWARGS_BLACKLIST}
+    _helper(base_ref, ref, entrypoint, live_update, restart_file, trigger, exit_policy, **cleaned_kwargs)
+
+
+def custom_build_with_restart(ref, command, deps, entrypoint, live_update,
+                              base_suffix='-tilt_docker_build_with_restart_base', restart_file=RESTART_FILE,
+                              trigger=None, exit_policy='restart', **kwargs):
+    """Wrap a custom_build call and its associated live_update steps so that the last step
+    of any live update is to rerun the given entrypoint.
+
+     Args:
+      ref: name for this image (e.g. 'myproj/backend' or 'myregistry/myproj/backend'); as the parameter of the same name in custom_build
+      command: build command for building your image
+      deps: source dependencies of the custom build
+      entrypoint: the command to be (re-)executed when the container starts or when a live_update is run
+      live_update: set of steps for updating a running container; as the parameter of the same name in custom_build
+      base_suffix: suffix for naming the base image, applied as {ref}{base_suffix}
+      restart_file: file that Tilt will update during a live_update to signal the entrypoint to rerun
+      trigger: (optional) list of local paths. If specified, the process will ONLY be restarted when there are changes
+               to the given file(s); as the parameter of the same name in the LiveUpdate `run` step.
+      **kwargs: will be passed to the underlying `custom_build` call
+    """
+
+    # first, validate the given live_update steps
+    if len(live_update) == 0:
+        fail("`custom_build_with_restart` requires at least one live_update step")
+    for step in live_update:
+        if type(step) == TYPE_RESTART_CONTAINER_STEP:
+            fail("`custom_build_with_restart` is not compatible with live_update step: "+
+                 "`restart_container()` (this extension is meant to REPLACE restart_container() )")
+
+    for k, v in kwargs.items():
+        if k == 'skips_local_docker':
+            fail("`custom_build_with_restart` is not compatible with `skips_local_docker`, because it needs access to the image")
+        if k == 'disable_push':
+            fail("`custom_build_with_restart` is not compatible with `disable_push`")
+        if k == 'tag':
+            fail("`custom_build_with_restart` renames your base image, so is not compatible with `tag`")
+
+    # rename the original image to make it a base image and declare a custom_build for it
+    base_ref = '{}{}'.format(ref, base_suffix)
+    custom_build(base_ref, command=command, deps=deps, **kwargs)
+
+    # A few arguments aren't applicable to the docker_build, so remove them.
+    cleaned_kwargs = {k: v for k, v in kwargs.items() if k not in _CUSTOM_BUILD_KWARGS_BLACKLIST}
+    _helper(base_ref, ref, entrypoint, live_update, restart_file, trigger, exit_policy, **cleaned_kwargs)
diff --git a/tilt_modules/restart_process/release.sh b/tilt_modules/restart_process/release.sh
new file mode 100755
index 0000000..b236822
--- /dev/null
+++ b/tilt_modules/restart_process/release.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -ex
+
+TIMESTAMP=$(date +'%Y-%m-%d')
+IMAGE_NAME='tiltdev/restart-helper'
+IMAGE_WITH_TAG=$IMAGE_NAME:$TIMESTAMP
+
+# build binary for tilt-restart-wrapper
+env GOOS=linux GOARCH=amd64 go build tilt-restart-wrapper.go
+
+# build Docker image with static binaries of:
+# - tilt-restart-wrapper (compiled above)
+# - entr (dependency of tilt-restart-wrapper)
+docker build . -t "$IMAGE_NAME"
+docker push "$IMAGE_NAME"
+
+docker tag "$IMAGE_NAME" "$IMAGE_WITH_TAG"
+docker push "$IMAGE_WITH_TAG"
+
+echo "Successfully built and pushed $IMAGE_WITH_TAG"
+
+
+
diff --git a/tilt_modules/restart_process/test/Dockerfile b/tilt_modules/restart_process/test/Dockerfile
new file mode 100644
index 0000000..95934f1
--- /dev/null
+++ b/tilt_modules/restart_process/test/Dockerfile
@@ -0,0 +1,5 @@
+FROM busybox
+
+COPY fail.sh /
+
+ENTRYPOINT /fail.sh
diff --git a/tilt_modules/restart_process/test/Tiltfile b/tilt_modules/restart_process/test/Tiltfile
new file mode 100644
index 0000000..5a943f9
--- /dev/null
+++ b/tilt_modules/restart_process/test/Tiltfile
@@ -0,0 +1,5 @@
+load('../Tiltfile', 'docker_build_with_restart')
+
+k8s_yaml('job.yaml')
+docker_build_with_restart('failing_job', '.', '/fail.sh',
+                          live_update=[sync('./fail.sh', '/fail.sh')])
diff --git a/tilt_modules/restart_process/test/custom.Tiltfile b/tilt_modules/restart_process/test/custom.Tiltfile
new file mode 100644
index 0000000..692a9bc
--- /dev/null
+++ b/tilt_modules/restart_process/test/custom.Tiltfile
@@ -0,0 +1,9 @@
+load('../Tiltfile', 'custom_build_with_restart')
+
+k8s_yaml('job.yaml')
+custom_build_with_restart(
+  'failing_job',
+  command='docker build -t $EXPECTED_REF .',
+  deps=['fail.sh'],
+  entrypoint='/fail.sh',
+  live_update=[sync('./fail.sh', '/fail.sh')])
diff --git a/tilt_modules/restart_process/test/fail.sh b/tilt_modules/restart_process/test/fail.sh
new file mode 100755
index 0000000..1e51f5e
--- /dev/null
+++ b/tilt_modules/restart_process/test/fail.sh
@@ -0,0 +1,9 @@
+#! /bin/sh
+
+echo "Are you there, pod?"
+sleep 1
+
+# Exit with a non-zero status code; we check that docker_build_with_restart
+# surfaces this error code to k8s, so k8s knows that the job failed.
+echo "Exiting with status code 123 😱"
+exit 123
diff --git a/tilt_modules/restart_process/test/job.yaml b/tilt_modules/restart_process/test/job.yaml
new file mode 100644
index 0000000..09e5445
--- /dev/null
+++ b/tilt_modules/restart_process/test/job.yaml
@@ -0,0 +1,12 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: failing-job
+spec:
+  template:
+    spec:
+      containers:
+        - name: failing-job
+          image: failing_job
+      restartPolicy: Never
+  backoffLimit: 4
diff --git a/tilt_modules/restart_process/test/test-custom.sh b/tilt_modules/restart_process/test/test-custom.sh
new file mode 100755
index 0000000..e06b5da
--- /dev/null
+++ b/tilt_modules/restart_process/test/test-custom.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+cd "$(dirname "$0")" || exit 1
+
+set -x
+tilt ci -f custom.Tiltfile > tilt.log 2>&1
+CI_EXIT=$?
+
+tilt down
+
+if [ $CI_EXIT -eq 0 ]; then
+  echo "Expected 'tilt ci' to fail, but succeeded."
+  exit 1
+fi
+
+grep -q "Are you there, pod?" tilt.log
+GREP_EXIT=$?
+
+rm tilt.log
+
+exit $GREP_EXIT
diff --git a/tilt_modules/restart_process/test/test-docker.sh b/tilt_modules/restart_process/test/test-docker.sh
new file mode 100755
index 0000000..63c50ce
--- /dev/null
+++ b/tilt_modules/restart_process/test/test-docker.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Test case for https://github.com/tilt-dev/tilt-extensions/issues/92
+#
+# This job will always exit with a non-zero status code; make sure
+# that docker_build_with_restart surfaces this error code to k8s,
+# so k8s knows that the job failed. (Thus, we expect the `tilt ci`
+# call to fail.)
+cd "$(dirname "$0")" || exit 1
+
+set -x
+tilt ci > tilt.log 2>&1
+CI_EXIT=$?
+
+tilt down
+
+if [ $CI_EXIT -eq 0 ]; then
+  echo "Expected 'tilt ci' to fail, but succeeded."
+  exit 1
+fi
+
+grep -q "Are you there, pod?" tilt.log
+GREP_EXIT=$?
+
+cat tilt.log
+
+rm tilt.log
+
+exit $GREP_EXIT
diff --git a/tilt_modules/restart_process/test/test.sh b/tilt_modules/restart_process/test/test.sh
new file mode 100755
index 0000000..2705502
--- /dev/null
+++ b/tilt_modules/restart_process/test/test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ex
+
+cd "$(dirname "$0")"
+./test-docker.sh
+./test-custom.sh
diff --git a/tilt_modules/restart_process/tilt-restart-wrapper.go b/tilt_modules/restart_process/tilt-restart-wrapper.go
new file mode 100644
index 0000000..ddc81b3
--- /dev/null
+++ b/tilt_modules/restart_process/tilt-restart-wrapper.go
@@ -0,0 +1,74 @@
+// `tilt-restart-wrapper` wraps `entr` (http://eradman.com/entrproject/) to easily
+// rerun a user-specified command when a given file changes.
+//
+// This is Tilt's recommended way of restarting a process as part of a live_update:
+// if your container invokes your app via the restart wrapper (e.g. `tilt-restart-wrapper /bin/my-app`),
+// you can trigger re-execution of your app with a live_update `run` step that makes
+// a trivial change to the file watched by `entr` (e.g. `run('date > /.restart-proc')`)
+//
+// This script exists (i.e. we're wrapping `entr` in a binary instead of invoking
+// it directly) because in its canonical invocation, `entr` requires that the
+// file(s) to watch be piped via stdin, i.e. it is invoked like:
+//     echo "/.restart-proc" | entr -rz /bin/my-app
+//
+// When specified as a `command` in Kubernetes or Docker Compose YAML (this is how
+// Tilt overrides entrypoints), the above would therefore need to be executed as shell:
+//     /bin/sh -c 'echo "/.restart-proc" | entr -rz /bin/my-app'
+
+// Any args specified in Kubernetes or Docker Compose YAML are attached to the end
+// of this call, and therefore in this case apply TO THE `/bin/sh -c` CALL, rather
+// than to the actual command run by `entr`; that is, any `args` specified by the
+// user would be effectively ignored.
+//
+// In order to make `entr` executable as ARGV rather than as shell, we have wrapped it
+// in a binary that can be called directly and takes care of the piping under the hood.
+//
+// Note: ideally `entr` could accept files-to-watch via flag instead of stdin,
+// but (for a number of good reasons) this feature isn't likely to be added any
+// time soon (see https://github.com/eradman/entr/issues/33).
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+)
+
+var watchFile = flag.String("watch_file", "/.restart-proc", "File that entr will watch for changes; changes to this file trigger `entr` to rerun the command(s) passed")
+var entrPath = flag.String("entr_path", "/entr", "Path to `entr` executable")
+var entrFlags = flag.String("entr_flags", "-rz", "Command line flags to pass to `entr` executable")
+
+func main() {
+	flag.Parse()
+
+	cmd := exec.Command(*entrPath, *entrFlags)
+	cmd.Stdin = strings.NewReader(fmt.Sprintf("%s\n", *watchFile))
+	cmd.Args = append(cmd.Args, flag.Args()...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Run(); err != nil {
+		if exiterr, ok := err.(*exec.ExitError); ok {
+			// The program has exited with an exit code != 0
+			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+				if len(flag.Args()) == 0 {
+					log.Println("`tilt-restart-wrapper` requires at least one positional arg " +
+						"(a command or set of args to be  executed / rerun whenever `watch_file` changes)")
+				}
+				os.Exit(status.ExitStatus())
+			}
+		} else {
+			log.Fatalf("error running command: %v", err)
+		}
+	}
+
+	if len(flag.Args()) == 0 {
+		log.Fatal("`tilt-restart-wrapper` requires at least one positional arg " +
+			"(will be passed to `entr` and executed / rerun whenever `watch_file` changes)")
+	}
+}

From 0db24349fdc25cab75502eef98398d62da9c4d5c Mon Sep 17 00:00:00 2001
From: Luke Van Seters <lukevanseters@gmail.com>
Date: Mon, 13 Sep 2021 20:36:09 -0400
Subject: [PATCH 07/11] print => logging

---
 mev_inspect/inspect_block.py | 15 ++++++++++-----
 run.py                       |  5 +++--
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/mev_inspect/inspect_block.py b/mev_inspect/inspect_block.py
index c6e04af..507ed3b 100644
--- a/mev_inspect/inspect_block.py
+++ b/mev_inspect/inspect_block.py
@@ -1,3 +1,5 @@
+import logging
+
 from web3 import Web3
 
 from mev_inspect.arbitrages import get_arbitrages
@@ -20,6 +22,9 @@ from mev_inspect.miner_payments import get_miner_payments
 from mev_inspect.swaps import get_swaps
 
 
+logger = logging.getLogger(__name__)
+
+
 def inspect_block(
     db_session,
     base_provider,
@@ -33,30 +38,30 @@ def inspect_block(
 ):
     block = create_from_block_number(base_provider, w3, block_number, should_cache)
 
-    print(f"Total traces: {len(block.traces)}")
+    logger.info(f"Total traces: {len(block.traces)}")
 
     total_transactions = len(
         set(t.transaction_hash for t in block.traces if t.transaction_hash is not None)
     )
-    print(f"Total transactions: {total_transactions}")
+    logger.info(f"Total transactions: {total_transactions}")
 
     trace_clasifier = TraceClassifier()
     classified_traces = trace_clasifier.classify(block.traces)
-    print(f"Returned {len(classified_traces)} classified traces")
+    logger.info(f"Returned {len(classified_traces)} classified traces")
 
     if should_write_classified_traces:
         delete_classified_traces_for_block(db_session, block_number)
         write_classified_traces(db_session, classified_traces)
 
     swaps = get_swaps(classified_traces)
-    print(f"Found {len(swaps)} swaps")
+    logger.info(f"Found {len(swaps)} swaps")
 
     if should_write_swaps:
         delete_swaps_for_block(db_session, block_number)
         write_swaps(db_session, swaps)
 
     arbitrages = get_arbitrages(swaps)
-    print(f"Found {len(arbitrages)} arbitrages")
+    logger.info(f"Found {len(arbitrages)} arbitrages")
 
     if should_write_arbitrages:
         delete_arbitrages_for_block(db_session, block_number)
diff --git a/run.py b/run.py
index 22b542a..5938f8e 100644
--- a/run.py
+++ b/run.py
@@ -2,6 +2,7 @@ import logging
 import signal
 import time
 
+
 logging.basicConfig(filename="app.log", level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
@@ -21,8 +22,8 @@ class GracefulKiller:
 
     def exit_gracefully(self, signum, frame):  # pylint: disable=unused-argument
         signal_name = self.signal_names[signum]
-        print(f"Received {signal_name} signal")
-        print("Cleaning up resources. End of process")
+        logger.info(f"Received {signal_name} signal")
+        logger.info("Cleaning up resources. End of process")
         self.kill_now = True
 
 

From e6793ee053be097ba265e798490019e99739f571 Mon Sep 17 00:00:00 2001
From: Luke Van Seters <lukevanseters@gmail.com>
Date: Mon, 13 Sep 2021 20:59:52 -0400
Subject: [PATCH 08/11] Add configmap for RPC. Print latest block on loop

---
 Tiltfile                               |   5 ++
 k8s/app.yaml                           |   5 ++
 mev_inspect/block.py                   |   4 +
 mev_inspect/provider.py                |  14 ++++
 run.py                                 |  18 +++-
 scripts/inspect_commands.py            |  17 +---
 tilt_modules/configmap/README.md       |  54 ++++++++++++
 tilt_modules/configmap/Tiltfile        | 109 +++++++++++++++++++++++++
 tilt_modules/configmap/test/Tiltfile   |   7 ++
 tilt_modules/configmap/test/job.yaml   |  29 +++++++
 tilt_modules/configmap/test/my-job.env |   1 +
 tilt_modules/configmap/test/my-job.ini |   1 +
 tilt_modules/configmap/test/test.sh    |   7 ++
 tilt_modules/extensions.json           |   5 ++
 14 files changed, 260 insertions(+), 16 deletions(-)
 create mode 100644 mev_inspect/provider.py
 create mode 100644 tilt_modules/configmap/README.md
 create mode 100644 tilt_modules/configmap/Tiltfile
 create mode 100644 tilt_modules/configmap/test/Tiltfile
 create mode 100644 tilt_modules/configmap/test/job.yaml
 create mode 100644 tilt_modules/configmap/test/my-job.env
 create mode 100644 tilt_modules/configmap/test/my-job.ini
 create mode 100755 tilt_modules/configmap/test/test.sh

diff --git a/Tiltfile b/Tiltfile
index abc17cb..10714ab 100644
--- a/Tiltfile
+++ b/Tiltfile
@@ -1,6 +1,7 @@
 load('ext://helm_remote', 'helm_remote')
 load('ext://restart_process', 'docker_build_with_restart')
 load('ext://secret', 'secret_from_dict')
+load('ext://configmap', 'configmap_from_dict')
 
 helm_remote("postgresql",
             repo_name='bitnami',
@@ -8,6 +9,10 @@ helm_remote("postgresql",
             set=["postgresqlPassword=password", "postgresqlDatabase=mev_inspect"],
 )
 
+k8s_yaml(configmap_from_dict("mev-inspect-rpc", inputs = {
+    "url" : os.environ["RPC_URL"],
+}))
+
 k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = {
     "username" : "postgres",
     "password": "password",
diff --git a/k8s/app.yaml b/k8s/app.yaml
index da45f2e..d5cabc1 100644
--- a/k8s/app.yaml
+++ b/k8s/app.yaml
@@ -31,6 +31,11 @@ spec:
               key: password
         - name: POSTGRES_HOST
           value: postgresql
+        - name: RPC_URL
+          valueFrom:
+            configMapKeyRef:
+              name: mev-inspect-rpc
+              key: url
         livenessProbe:
           exec:
             command:
diff --git a/mev_inspect/block.py b/mev_inspect/block.py
index dca522d..c05b16c 100644
--- a/mev_inspect/block.py
+++ b/mev_inspect/block.py
@@ -11,6 +11,10 @@ from mev_inspect.schemas.receipts import Receipt
 cache_directory = "./cache"
 
 
+def get_latest_block_number(w3: Web3) -> int:
+    return w3.eth.get_block("latest")["number"]
+
+
 def create_from_block_number(
     base_provider, w3: Web3, block_number: int, should_cache: bool
 ) -> Block:
diff --git a/mev_inspect/provider.py b/mev_inspect/provider.py
new file mode 100644
index 0000000..1bdc68d
--- /dev/null
+++ b/mev_inspect/provider.py
@@ -0,0 +1,14 @@
+from web3 import Web3
+
+from mev_inspect.retry import http_retry_with_backoff_request_middleware
+
+
+def get_base_provider(rpc: str) -> Web3.HTTPProvider:
+    base_provider = Web3.HTTPProvider(rpc)
+    base_provider.middlewares.remove("http_retry_request")
+    base_provider.middlewares.add(
+        http_retry_with_backoff_request_middleware,
+        "http_retry_with_backoff",
+    )
+
+    return base_provider
diff --git a/run.py b/run.py
index 5938f8e..7a93d5e 100644
--- a/run.py
+++ b/run.py
@@ -1,9 +1,15 @@
 import logging
+import os
 import signal
 import time
 
+from web3 import Web3
 
-logging.basicConfig(filename="app.log", level=logging.DEBUG)
+from mev_inspect.block import get_latest_block_number
+from mev_inspect.provider import get_base_provider
+
+
+logging.basicConfig(filename="app.log", level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
@@ -28,11 +34,19 @@ class GracefulKiller:
 
 
 if __name__ == "__main__":
+    rpc = os.getenv("RPC_URL")
+    if rpc is None:
+        raise RuntimeError("Missing environment variable RPC_URL")
+
     logger.info("Starting...")
+
     killer = GracefulKiller()
+    base_provider = get_base_provider(rpc)
+    w3 = Web3(base_provider)
 
     while not killer.kill_now:
-        logger.info("Running...")
+        latest_block_number = get_latest_block_number(w3)
+        logger.info(f"Latest block: {latest_block_number}")
         time.sleep(5)
 
     logger.info("Stopping...")
diff --git a/scripts/inspect_commands.py b/scripts/inspect_commands.py
index ec38408..ceb2138 100644
--- a/scripts/inspect_commands.py
+++ b/scripts/inspect_commands.py
@@ -3,7 +3,7 @@ from web3 import Web3
 
 from mev_inspect.db import get_session
 from mev_inspect.inspect_block import inspect_block
-from mev_inspect.retry import http_retry_with_backoff_request_middleware
+from mev_inspect.provider import get_base_provider
 
 
 @click.group()
@@ -17,7 +17,7 @@ def cli():
 @click.option("--cache/--no-cache", default=True)
 def inspect_block_command(block_number: int, rpc: str, cache: bool):
     db_session = get_session()
-    base_provider = _get_base_provider(rpc)
+    base_provider = get_base_provider(rpc)
     w3 = Web3(base_provider)
 
     if not cache:
@@ -36,7 +36,7 @@ def inspect_many_blocks_command(
 ):
 
     db_session = get_session()
-    base_provider = _get_base_provider(rpc)
+    base_provider = get_base_provider(rpc)
     w3 = Web3(base_provider)
 
     if not cache:
@@ -61,16 +61,5 @@ def inspect_many_blocks_command(
         )
 
 
-def _get_base_provider(rpc: str) -> Web3.HTTPProvider:
-    base_provider = Web3.HTTPProvider(rpc)
-    base_provider.middlewares.remove("http_retry_request")
-    base_provider.middlewares.add(
-        http_retry_with_backoff_request_middleware,
-        "http_retry_with_backoff",
-    )
-
-    return base_provider
-
-
 if __name__ == "__main__":
     cli()
diff --git a/tilt_modules/configmap/README.md b/tilt_modules/configmap/README.md
new file mode 100644
index 0000000..bbed5eb
--- /dev/null
+++ b/tilt_modules/configmap/README.md
@@ -0,0 +1,54 @@
+# Configmap
+
+Author: [Nick Santos](https://github.com/nicks)
+
+Helper functions for creating Kubernetes configmaps.
+
+## Functions
+
+### configmap_yaml
+
+```
+configmap_yaml(name: str, namespace: str = "", from_file: Union[str, List[str]] = None, watch: bool = True, from_env_file: str = None): Blob
+```
+
+Returns YAML for a config map generated from a file.
+
+* `from_file` ( str ) – equivalent to `kubectl create configmap --from-file`
+* `from_env_file` (str) - equivalent to `kubectl create configmap --from-env-file`
+* `watch` ( bool ) - auto-reload if the files change
+
+### configmap_create
+
+```
+configmap_create(name: str, namespace: str = "", from_file: Union[str, List[str]] = None, watch: bool = True, from_env_file: str = None)
+```
+
+Deploys a config map. Equivalent to
+
+```
+k8s_yaml(configmap_yaml('name', from_file=[...]))
+```
+
+### configmap_from_dict
+
+```
+configmap_from_dict(name: str, namespace: str = "", inputs: Dict[str, str]] = None): Blob
+```
+
+Returns YAML for a config map generated from a given dictionary. Nested dictionaries are not supported
+
+* `inputs` ( dict ) – equivalent to `kubectl create configmap --from-literal` for each key and value
+
+## Example Usage
+
+### For a Grafana config
+
+```
+load('ext://configmap', 'configmap_create')
+configmap_create('grafana-config', from_file=['grafana.ini=./grafana.ini'])
+```
+
+## Caveats
+
+- This extension doesn't do any validation to confirm that names or namespaces are valid.
diff --git a/tilt_modules/configmap/Tiltfile b/tilt_modules/configmap/Tiltfile
new file mode 100644
index 0000000..71bc9cd
--- /dev/null
+++ b/tilt_modules/configmap/Tiltfile
@@ -0,0 +1,109 @@
+# -*- mode: Python -*-
+
+def configmap_yaml(name, namespace="", from_file=None, watch=True, from_env_file=None):
+  """Returns YAML for a generic configmap
+
+  Args:
+    name: The configmap name.
+    namespace: The namespace.
+    from_file: Use the from-file configmap generator. May be a string or a list of strings.
+       Example: ["grafana.ini=path/to/grafana.ini"]
+    watch: Reruns the Tiltfile and re-deploys automatically if the from-files change.
+       Defaults to true.
+    from_env_file: Use from-env-file configmap generator. Must be string.
+       Example: "./local.env"
+
+  Returns:
+    The configmap YAML as a blob
+  """
+
+  args = [
+    "kubectl",
+    "create",
+    "configmap",
+    name,
+  ]
+
+  if namespace:
+    args.extend(["-n", namespace])
+
+  generator = False
+
+  if from_file and from_env_file:
+    fail("Must specify either 'from_file' OR 'from_env_file'")
+
+  if from_file:
+    if type(from_file) == "string":
+      from_file = [from_file]
+
+    if type(from_file) == "list":
+      for f in from_file:
+        args.extend(["--from-file", f])
+        if watch:
+          l = f.split('=')
+          watch_file(l[len(l)-1])
+      generator = True
+    else:
+      fail("Bad from_file argument: %s" % from_file)
+  elif from_env_file:
+    if type(from_env_file) == "list":
+      fail("from_env_file only supports string as an input to prevent confusion with kubectl behavior of only loading the last item in a list")
+    elif type(from_env_file == "string"):
+      args.extend(["--from-env-file", from_env_file])
+      if watch:
+        watch_file(from_env_file)
+      generator = True
+
+  if not generator:
+    fail("No configmap generator specified")
+
+  args.extend(["-o=yaml", "--dry-run=client"])
+  return local(args, quiet=True)
+
+def configmap_from_dict(name, namespace="", inputs={}):
+    """Returns YAML for a generic configmap
+    Args:
+        name: The configmap name.
+        namespace: The namespace.
+        inputs: A dict of keys and values to use. Nesting is not supported
+    Returns:
+        The configmap YAML as a blob
+    """
+
+    args = [
+        "kubectl",
+        "create",
+        "configmap",
+        name,
+    ]
+
+    if namespace:
+        args.extend(["-n", namespace])
+
+    if type(inputs) != "dict":
+        fail("Bad argument to configmap_from_dict, inputs was not dict typed")
+
+    for k,v in inputs.items():
+        args.extend(["--from-literal", "%s=%s" % (k,v)])
+
+    args.extend(["-o=yaml", "--dry-run=client"])
+    return local(args, quiet=True)
+
+def configmap_create(name, namespace="", from_file=None, watch=True, from_env_file=None):
+  """Creates a configmap in the current Kubernetes cluster.
+
+  Generators:
+    - from_file: Wraps kubectl from-file behavior.
+    - from_env_file: Wraps kubectl from-env-file behavior.
+
+  Args:
+    name: The configmap name.
+    namespace: The namespace.
+    from_file: Use the from-file configmap generator. May be a string or a list of strings.
+       Example: ["grafana.ini=path/to/grafana.ini"]
+    watch: Reruns the Tiltfile and re-deploys automatically if the from-files change.
+       Defaults to true.
+    from_env_file: Use from-env-file configmap generator. Must be string.
+       Example: "./local.env"
+  """
+  k8s_yaml(configmap_yaml(name, namespace, from_file, watch, from_env_file))
diff --git a/tilt_modules/configmap/test/Tiltfile b/tilt_modules/configmap/test/Tiltfile
new file mode 100644
index 0000000..1e7b9dc
--- /dev/null
+++ b/tilt_modules/configmap/test/Tiltfile
@@ -0,0 +1,7 @@
+load('../Tiltfile', 'configmap_create', 'configmap_from_dict')
+
+configmap_create('job-config', from_file='my-job.ini=./my-job.ini')
+configmap_create('env-job-config', from_env_file='my-job.env')
+configmap_from_dict('from-dict-config', inputs={"hello":"world"})
+
+k8s_yaml('job.yaml')
\ No newline at end of file
diff --git a/tilt_modules/configmap/test/job.yaml b/tilt_modules/configmap/test/job.yaml
new file mode 100644
index 0000000..1d43252
--- /dev/null
+++ b/tilt_modules/configmap/test/job.yaml
@@ -0,0 +1,29 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: configmap-verify
+spec:
+  backoffLimit: 1
+  template:
+    spec:
+      containers:
+      - name: configmap-env-verify
+        image: alpine
+        command: ["/bin/echo", "$(TEST_VAR)"]
+        env:
+          - name: TEST_VAR
+            valueFrom:
+              configMapKeyRef:
+                name: env-job-config
+                key: TEST_VAR
+      - name: configmap-verify
+        image: alpine
+        command: ["cat", "/etc/my-job/my-job.ini"]
+        volumeMounts:
+        - name: job-config
+          mountPath: /etc/my-job
+      restartPolicy: Never
+      volumes:
+      - name: job-config
+        configMap:
+          name: job-config
diff --git a/tilt_modules/configmap/test/my-job.env b/tilt_modules/configmap/test/my-job.env
new file mode 100644
index 0000000..c3b8b31
--- /dev/null
+++ b/tilt_modules/configmap/test/my-job.env
@@ -0,0 +1 @@
+TEST_VAR="hello-env!"
\ No newline at end of file
diff --git a/tilt_modules/configmap/test/my-job.ini b/tilt_modules/configmap/test/my-job.ini
new file mode 100644
index 0000000..3462721
--- /dev/null
+++ b/tilt_modules/configmap/test/my-job.ini
@@ -0,0 +1 @@
+hello!
\ No newline at end of file
diff --git a/tilt_modules/configmap/test/test.sh b/tilt_modules/configmap/test/test.sh
new file mode 100755
index 0000000..a49e009
--- /dev/null
+++ b/tilt_modules/configmap/test/test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cd "$(dirname "$0")"
+
+set -ex
+tilt ci
+tilt down
diff --git a/tilt_modules/extensions.json b/tilt_modules/extensions.json
index 4d58ca7..df6e050 100644
--- a/tilt_modules/extensions.json
+++ b/tilt_modules/extensions.json
@@ -19,6 +19,11 @@
       "Name": "restart_process",
       "ExtensionRegistry": "https://github.com/tilt-dev/tilt-extensions",
       "TimeFetched": "2021-09-13T20:14:11.011803-04:00"
+    },
+    {
+      "Name": "configmap",
+      "ExtensionRegistry": "https://github.com/tilt-dev/tilt-extensions",
+      "TimeFetched": "2021-09-13T20:58:06.169124-04:00"
     }
   ]
 }
\ No newline at end of file

From 50d04a0b423da88c95df5d94369823b0e9708c64 Mon Sep 17 00:00:00 2001
From: Luke Van Seters <lukevanseters@gmail.com>
Date: Mon, 13 Sep 2021 21:09:00 -0400
Subject: [PATCH 09/11] Use last written miner payment block as max written
 block

---
 mev_inspect/crud/miner_payments.py | 12 +++++++++++-
 run.py                             | 13 ++++++++++++-
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/mev_inspect/crud/miner_payments.py b/mev_inspect/crud/miner_payments.py
index e82bf96..afef55d 100644
--- a/mev_inspect/crud/miner_payments.py
+++ b/mev_inspect/crud/miner_payments.py
@@ -1,5 +1,7 @@
 import json
-from typing import List
+from typing import List, Optional
+
+from sqlalchemy.sql.expression import func
 
 from mev_inspect.models.miner_payments import MinerPaymentModel
 from mev_inspect.schemas.miner_payments import MinerPayment
@@ -29,3 +31,11 @@ def write_miner_payments(
 
     db_session.bulk_save_objects(models)
     db_session.commit()
+
+
+def get_max_miner_payment_block(db_session) -> Optional[int]:
+    results = db_session.query(func.max(MinerPaymentModel.block_number)).one_or_none()
+    if results is None:
+        return None
+    else:
+        return int(results[0])
diff --git a/run.py b/run.py
index 7a93d5e..01ba2d3 100644
--- a/run.py
+++ b/run.py
@@ -6,6 +6,8 @@ import time
 from web3 import Web3
 
 from mev_inspect.block import get_latest_block_number
+from mev_inspect.crud.miner_payments import get_max_miner_payment_block
+from mev_inspect.db import get_session
 from mev_inspect.provider import get_base_provider
 
 
@@ -33,7 +35,7 @@ class GracefulKiller:
         self.kill_now = True
 
 
-if __name__ == "__main__":
+def run():
     rpc = os.getenv("RPC_URL")
     if rpc is None:
         raise RuntimeError("Missing environment variable RPC_URL")
@@ -41,12 +43,21 @@ if __name__ == "__main__":
     logger.info("Starting...")
 
     killer = GracefulKiller()
+
+    db_session = get_session()
     base_provider = get_base_provider(rpc)
     w3 = Web3(base_provider)
 
     while not killer.kill_now:
         latest_block_number = get_latest_block_number(w3)
+        last_written_block = get_max_miner_payment_block(db_session)
+
         logger.info(f"Latest block: {latest_block_number}")
+        logger.info(f"Last written block: {last_written_block}")
         time.sleep(5)
 
     logger.info("Stopping...")
+
+
+if __name__ == "__main__":
+    run()

From a9cbe106ad46a98ecfd84c839a0c5275775927f6 Mon Sep 17 00:00:00 2001
From: Luke Van Seters <lukevanseters@gmail.com>
Date: Mon, 13 Sep 2021 21:55:19 -0400
Subject: [PATCH 10/11] Use a dedicated table for the last block written. Write
 new blocks as they come

---
 mev_inspect/block.py               |  2 +-
 mev_inspect/crud/miner_payments.py | 12 +---------
 run.py                             | 35 +++++++++++++++++++++++++-----
 3 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/mev_inspect/block.py b/mev_inspect/block.py
index c05b16c..1b974e4 100644
--- a/mev_inspect/block.py
+++ b/mev_inspect/block.py
@@ -12,7 +12,7 @@ cache_directory = "./cache"
 
 
 def get_latest_block_number(w3: Web3) -> int:
-    return w3.eth.get_block("latest")["number"]
+    return int(w3.eth.get_block("latest")["number"])
 
 
 def create_from_block_number(
diff --git a/mev_inspect/crud/miner_payments.py b/mev_inspect/crud/miner_payments.py
index afef55d..e82bf96 100644
--- a/mev_inspect/crud/miner_payments.py
+++ b/mev_inspect/crud/miner_payments.py
@@ -1,7 +1,5 @@
 import json
-from typing import List, Optional
-
-from sqlalchemy.sql.expression import func
+from typing import List
 
 from mev_inspect.models.miner_payments import MinerPaymentModel
 from mev_inspect.schemas.miner_payments import MinerPayment
@@ -31,11 +29,3 @@ def write_miner_payments(
 
     db_session.bulk_save_objects(models)
     db_session.commit()
-
-
-def get_max_miner_payment_block(db_session) -> Optional[int]:
-    results = db_session.query(func.max(MinerPaymentModel.block_number)).one_or_none()
-    if results is None:
-        return None
-    else:
-        return int(results[0])
diff --git a/run.py b/run.py
index 01ba2d3..173a1fc 100644
--- a/run.py
+++ b/run.py
@@ -6,8 +6,12 @@ import time
 from web3 import Web3
 
 from mev_inspect.block import get_latest_block_number
-from mev_inspect.crud.miner_payments import get_max_miner_payment_block
+from mev_inspect.crud.latest_block_update import (
+    find_latest_block_update,
+    update_latest_block,
+)
 from mev_inspect.db import get_session
+from mev_inspect.inspect_block import inspect_block
 from mev_inspect.provider import get_base_provider
 
 
@@ -48,13 +52,34 @@ def run():
     base_provider = get_base_provider(rpc)
     w3 = Web3(base_provider)
 
-    while not killer.kill_now:
-        latest_block_number = get_latest_block_number(w3)
-        last_written_block = get_max_miner_payment_block(db_session)
+    latest_block_number = get_latest_block_number(w3)
 
+    while not killer.kill_now:
+        last_written_block = find_latest_block_update(db_session)
         logger.info(f"Latest block: {latest_block_number}")
         logger.info(f"Last written block: {last_written_block}")
-        time.sleep(5)
+
+        if last_written_block is None or last_written_block < latest_block_number:
+            block_number = (
+                latest_block_number
+                if last_written_block is None
+                else last_written_block + 1
+            )
+
+            logger.info(f"Writing block: {block_number}")
+
+            inspect_block(
+                db_session,
+                base_provider,
+                w3,
+                block_number,
+                should_write_classified_traces=False,
+                should_cache=False,
+            )
+            update_latest_block(db_session, block_number)
+        else:
+            latest_block_number = get_latest_block_number(w3)
+            time.sleep(5)
 
     logger.info("Stopping...")
 

From b8280f84648949736245b06c78ad9b9cfb6585d1 Mon Sep 17 00:00:00 2001
From: Luke Van Seters <lukevanseters@gmail.com>
Date: Tue, 14 Sep 2021 12:06:20 -0400
Subject: [PATCH 11/11] Sleep first to get newest block after sleep

---
 run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/run.py b/run.py
index 173a1fc..2435dff 100644
--- a/run.py
+++ b/run.py
@@ -78,8 +78,8 @@ def run():
             )
             update_latest_block(db_session, block_number)
         else:
-            latest_block_number = get_latest_block_number(w3)
             time.sleep(5)
+            latest_block_number = get_latest_block_number(w3)
 
     logger.info("Stopping...")