Merge pull request #103 from flashbots/backfill-main

Add support for parallelized backfills

backfill.py

@@ -0,0 +1,57 @@
+import subprocess
+import sys
+from typing import Iterator, Tuple
+
+
+def get_block_after_before_chunks(
+    after_block: int,
+    before_block: int,
+    n_workers: int,
+) -> Iterator[Tuple[int, int]]:
+    n_blocks = before_block - after_block
+    remainder = n_blocks % n_workers
+    floor_chunk_size = n_blocks // n_workers
+
+    last_before_block = None
+
+    for worker_index in range(n_workers):
+        chunk_size = floor_chunk_size
+
+        if worker_index < remainder:
+            chunk_size += 1
+
+        batch_after_block = (
+            last_before_block if last_before_block is not None else after_block
+        )
+
+        batch_before_block = batch_after_block + chunk_size
+        yield batch_after_block, batch_before_block
+        last_before_block = batch_before_block
+
+
+def backfill(after_block: int, before_block: int, n_workers: int):
+    if n_workers <= 0:
+        raise ValueError("Need at least one worker")
+
+    for batch_after_block, batch_before_block in get_block_after_before_chunks(
+        after_block,
+        before_block,
+        n_workers,
+    ):
+        print(f"Backfilling {batch_after_block} to {batch_before_block}")
+        backfill_command = f"sh backfill.sh {batch_after_block} {batch_before_block}"
+        process = subprocess.Popen(backfill_command.split(), stdout=subprocess.PIPE)
+        output, _ = process.communicate()
+        print(output)
+
+
+def main():
+    after_block = int(sys.argv[1])
+    before_block = int(sys.argv[2])
+    n_workers = int(sys.argv[3])
+
+    backfill(after_block, before_block, n_workers)
+
+
+if __name__ == "__main__":
+    main()

backfill.sh

@@ -0,0 +1,6 @@
+current_image=$(kubectl get deployment mev-inspect -o=jsonpath='{$.spec.template.spec.containers[:1].image}')
+
+helm template mev-inspect-backfill ./k8s/mev-inspect-backfill \
+    --set image.repository=$current_image \
+    --set command.startBlockNumber=$1 \
+    --set command.endBlockNumber=$2 | kubectl apply -f -

k8s/mev-inspect-backfill/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

k8s/mev-inspect-backfill/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: mev-inspect-backfill
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

k8s/mev-inspect-backfill/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "mev-inspect-backfill.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mev-inspect-backfill.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "mev-inspect-backfill.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "mev-inspect-backfill.labels" -}}
+helm.sh/chart: {{ include "mev-inspect-backfill.chart" . }}
+{{ include "mev-inspect-backfill.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "mev-inspect-backfill.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "mev-inspect-backfill.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "mev-inspect-backfill.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "mev-inspect-backfill.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

k8s/mev-inspect-backfill/templates/job.yaml

@@ -0,0 +1,51 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "mev-inspect-backfill.fullname" . }}-{{ randAlphaNum 5 | lower }}
+  labels:
+    {{- include "mev-inspect-backfill.labels" . | nindent 4 }}
+spec:
+  completions: 1
+  parallelism: 1
+  ttlSecondsAfterFinished: 5
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command:
+            - poetry
+            - run
+            - inspect-many-blocks
+            - {{ .Values.command.startBlockNumber | quote }}
+            - {{ .Values.command.endBlockNumber | quote }}
+          env:
+          - name: POSTGRES_HOST
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: host
+          - name: POSTGRES_USER
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: username
+          - name: POSTGRES_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: password
+          - name: RPC_URL
+            valueFrom:
+              configMapKeyRef:
+                name: mev-inspect-rpc
+                key: url
+      restartPolicy: OnFailure

k8s/mev-inspect-backfill/values.yaml

@@ -0,0 +1,42 @@
+# Default values for mev-inspect.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+image:
+  repository: mev-inspect-py
+  pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

k8s/mev-inspect/templates/deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   labels:
     {{- include "mev-inspect.labels" . | nindent 4 }}
 spec:
-  replicas: 1
+  replicas: {{ .Values.replicaCount }}
   selector:
     matchLabels:
       {{- include "mev-inspect.selectorLabels" . | nindent 6 }}
@@ -28,7 +28,7 @@ spec:
         - name: {{ .Chart.Name }}
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          image: "{{ .Values.image.repository }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           livenessProbe:
             exec:

k8s/mev-inspect/values.yaml

@@ -2,10 +2,11 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
+replicaCount: 1
+
 image:
-  repository: mev-inspect-py
+  repository: mev-inspect-py:latest
   pullPolicy: IfNotPresent
-  tag: "latest"
 
 imagePullSecrets: []
 nameOverride: ""

mev

@@ -24,6 +24,14 @@ case "$1" in
         echo "Connecting to $DB_NAME"
         db
 	;;
+  backfill)
+        start_block_number=$2
+        end_block_number=$3
+        n_workers=$4
+
+        echo "Backfilling from $start_block_number to $end_block_number with $n_workers workers"
+        python backfill.py $start_block_number $end_block_number $n_workers
+	;;
   inspect)
         block_number=$2
         echo "Inspecting block $block_number"