From af75fbc35a5ad4ac4c83208ede890551b4440372 Mon Sep 17 00:00:00 2001
From: Luke Van Seters <lukevanseters@gmail.com>
Date: Thu, 14 Oct 2021 17:41:57 -0400
Subject: [PATCH] Add backfill scripts

---
 backfill.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 backfill.sh |  6 ++++++
 mev         |  8 ++++++++
 3 files changed, 71 insertions(+)
 create mode 100644 backfill.py
 create mode 100644 backfill.sh

diff --git a/backfill.py b/backfill.py
new file mode 100644
index 0000000..4c2dc2a
--- /dev/null
+++ b/backfill.py
@@ -0,0 +1,57 @@
+import subprocess
+import sys
+from typing import Iterator, Tuple
+
+
+def get_block_after_before_chunks(
+    after_block: int,
+    before_block: int,
+    n_workers: int,
+) -> Iterator[Tuple[int, int]]:
+    n_blocks = before_block - after_block
+    remainder = n_blocks % n_workers
+    floor_chunk_size = n_blocks // n_workers
+
+    last_before_block = None
+
+    for worker_index in range(n_workers):
+        chunk_size = floor_chunk_size
+
+        if worker_index < remainder:
+            chunk_size += 1
+
+        batch_after_block = (
+            last_before_block if last_before_block is not None else after_block
+        )
+
+        batch_before_block = batch_after_block + chunk_size
+        yield batch_after_block, batch_before_block
+        last_before_block = batch_before_block
+
+
+def backfill(after_block: int, before_block: int, n_workers: int):
+    if n_workers <= 0:
+        raise ValueError("Need at least one worker")
+
+    for batch_after_block, batch_before_block in get_block_after_before_chunks(
+        after_block,
+        before_block,
+        n_workers,
+    ):
+        print(f"Backfilling {batch_after_block} to {batch_before_block}")
+        backfill_command = f"sh backfill.sh {batch_after_block} {batch_before_block}"
+        process = subprocess.Popen(backfill_command.split(), stdout=subprocess.PIPE)
+        output, _ = process.communicate()
+        print(output)
+
+
+def main():
+    after_block = int(sys.argv[1])
+    before_block = int(sys.argv[2])
+    n_workers = int(sys.argv[3])
+
+    backfill(after_block, before_block, n_workers)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/backfill.sh b/backfill.sh
new file mode 100644
index 0000000..5cc06af
--- /dev/null
+++ b/backfill.sh
@@ -0,0 +1,6 @@
+current_image=$(kubectl get deployment mev-inspect -o=jsonpath='{$.spec.template.spec.containers[:1].image}')
+
+helm template mev-inspect-backfill ./k8s/mev-inspect-backfill \
+    --set image.repository=$current_image \
+    --set command.startBlockNumber=$1 \
+    --set command.endBlockNumber=$2 | kubectl apply -f -
diff --git a/mev b/mev
index 5351d9b..a936bf1 100755
--- a/mev
+++ b/mev
@@ -24,6 +24,14 @@ case "$1" in
         echo "Connecting to $DB_NAME"
         db
 	;;
+  backfill)
+        start_block_number=$2
+        end_block_number=$3
+        n_workers=$4
+
+        echo "Backfilling from $start_block_number to $end_block_number with $n_workers workers"
+        python backfill.py $start_block_number $end_block_number $n_workers
+	;;
   inspect)
         block_number=$2
         echo "Inspecting block $block_number"