diff --git a/backfill.py b/backfill.py new file mode 100644 index 0000000..4c2dc2a --- /dev/null +++ b/backfill.py @@ -0,0 +1,57 @@ +import subprocess +import sys +from typing import Iterator, Tuple + + +def get_block_after_before_chunks( + after_block: int, + before_block: int, + n_workers: int, +) -> Iterator[Tuple[int, int]]: + n_blocks = before_block - after_block + remainder = n_blocks % n_workers + floor_chunk_size = n_blocks // n_workers + + last_before_block = None + + for worker_index in range(n_workers): + chunk_size = floor_chunk_size + + if worker_index < remainder: + chunk_size += 1 + + batch_after_block = ( + last_before_block if last_before_block is not None else after_block + ) + + batch_before_block = batch_after_block + chunk_size + yield batch_after_block, batch_before_block + last_before_block = batch_before_block + + +def backfill(after_block: int, before_block: int, n_workers: int): + if n_workers <= 0: + raise ValueError("Need at least one worker") + + for batch_after_block, batch_before_block in get_block_after_before_chunks( + after_block, + before_block, + n_workers, + ): + print(f"Backfilling {batch_after_block} to {batch_before_block}") + backfill_command = f"sh backfill.sh {batch_after_block} {batch_before_block}" + process = subprocess.Popen(backfill_command.split(), stdout=subprocess.PIPE) + output, _ = process.communicate() + print(output) + + +def main(): + after_block = int(sys.argv[1]) + before_block = int(sys.argv[2]) + n_workers = int(sys.argv[3]) + + backfill(after_block, before_block, n_workers) + + +if __name__ == "__main__": + main() diff --git a/backfill.sh b/backfill.sh new file mode 100644 index 0000000..5cc06af --- /dev/null +++ b/backfill.sh @@ -0,0 +1,6 @@ +current_image=$(kubectl get deployment mev-inspect -o=jsonpath='{$.spec.template.spec.containers[:1].image}') + +helm template mev-inspect-backfill ./k8s/mev-inspect-backfill \ + --set image.repository=$current_image \ + --set command.startBlockNumber=$1 \ + --set command.endBlockNumber=$2 | kubectl apply -f - diff --git a/k8s/mev-inspect-backfill/.helmignore b/k8s/mev-inspect-backfill/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/k8s/mev-inspect-backfill/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s/mev-inspect-backfill/Chart.yaml b/k8s/mev-inspect-backfill/Chart.yaml new file mode 100644 index 0000000..5ecfa0e --- /dev/null +++ b/k8s/mev-inspect-backfill/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: mev-inspect-backfill +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/k8s/mev-inspect-backfill/templates/_helpers.tpl b/k8s/mev-inspect-backfill/templates/_helpers.tpl new file mode 100644 index 0000000..e3650e6 --- /dev/null +++ b/k8s/mev-inspect-backfill/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "mev-inspect-backfill.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "mev-inspect-backfill.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "mev-inspect-backfill.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "mev-inspect-backfill.labels" -}} +helm.sh/chart: {{ include "mev-inspect-backfill.chart" . }} +{{ include "mev-inspect-backfill.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "mev-inspect-backfill.selectorLabels" -}} +app.kubernetes.io/name: {{ include "mev-inspect-backfill.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "mev-inspect-backfill.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "mev-inspect-backfill.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/k8s/mev-inspect-backfill/templates/job.yaml b/k8s/mev-inspect-backfill/templates/job.yaml new file mode 100644 index 0000000..611d30a --- /dev/null +++ b/k8s/mev-inspect-backfill/templates/job.yaml @@ -0,0 +1,51 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "mev-inspect-backfill.fullname" . }}-{{ randAlphaNum 5 | lower }} + labels: + {{- include "mev-inspect-backfill.labels" . | nindent 4 }} +spec: + completions: 1 + parallelism: 1 + ttlSecondsAfterFinished: 5 + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - poetry + - run + - inspect-many-blocks + - {{ .Values.command.startBlockNumber | quote }} + - {{ .Values.command.endBlockNumber | quote }} + env: + - name: POSTGRES_HOST + valueFrom: + secretKeyRef: + name: mev-inspect-db-credentials + key: host + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: mev-inspect-db-credentials + key: username + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: mev-inspect-db-credentials + key: password + - name: RPC_URL + valueFrom: + configMapKeyRef: + name: mev-inspect-rpc + key: url + restartPolicy: OnFailure diff --git a/k8s/mev-inspect-backfill/values.yaml b/k8s/mev-inspect-backfill/values.yaml new file mode 100644 index 0000000..3e45e4e --- /dev/null +++ b/k8s/mev-inspect-backfill/values.yaml @@ -0,0 +1,42 @@ +# Default values for mev-inspect. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: mev-inspect-py + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/k8s/mev-inspect/templates/deployment.yaml b/k8s/mev-inspect/templates/deployment.yaml index 42e04d0..e06b7dd 100644 --- a/k8s/mev-inspect/templates/deployment.yaml +++ b/k8s/mev-inspect/templates/deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "mev-inspect.labels" . | nindent 4 }} spec: - replicas: 1 + replicas: {{ .Values.replicaCount }} selector: matchLabels: {{- include "mev-inspect.selectorLabels" . | nindent 6 }} @@ -28,7 +28,7 @@ spec: - name: {{ .Chart.Name }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.image.repository }}" imagePullPolicy: {{ .Values.image.pullPolicy }} livenessProbe: exec: diff --git a/k8s/mev-inspect/values.yaml b/k8s/mev-inspect/values.yaml index a34451e..11140f9 100644 --- a/k8s/mev-inspect/values.yaml +++ b/k8s/mev-inspect/values.yaml @@ -2,10 +2,11 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. +replicaCount: 1 + image: - repository: mev-inspect-py + repository: mev-inspect-py:latest pullPolicy: IfNotPresent - tag: "latest" imagePullSecrets: [] nameOverride: "" diff --git a/mev b/mev index 5351d9b..a936bf1 100755 --- a/mev +++ b/mev @@ -24,6 +24,14 @@ case "$1" in echo "Connecting to $DB_NAME" db ;; + backfill) + start_block_number=$2 + end_block_number=$3 + n_workers=$4 + + echo "Backfilling from $start_block_number to $end_block_number with $n_workers workers" + python backfill.py $start_block_number $end_block_number $n_workers + ;; inspect) block_number=$2 echo "Inspecting block $block_number"