From b33eb49dd2e6848a8bf5b95bfbfdb085e1d33b92 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 6 Jan 2022 17:10:52 -0500 Subject: [PATCH] Remove old backfill code --- backfill.py | 57 ---------------- backfill.sh | 6 -- k8s/mev-inspect-backfill/.helmignore | 23 ------- k8s/mev-inspect-backfill/Chart.yaml | 24 ------- .../templates/_helpers.tpl | 62 ----------------- k8s/mev-inspect-backfill/templates/job.yaml | 68 ------------------- k8s/mev-inspect-backfill/values.yaml | 42 ------------ 7 files changed, 282 deletions(-) delete mode 100644 backfill.py delete mode 100644 backfill.sh delete mode 100644 k8s/mev-inspect-backfill/.helmignore delete mode 100644 k8s/mev-inspect-backfill/Chart.yaml delete mode 100644 k8s/mev-inspect-backfill/templates/_helpers.tpl delete mode 100644 k8s/mev-inspect-backfill/templates/job.yaml delete mode 100644 k8s/mev-inspect-backfill/values.yaml diff --git a/backfill.py b/backfill.py deleted file mode 100644 index 4c2dc2a..0000000 --- a/backfill.py +++ /dev/null @@ -1,57 +0,0 @@ -import subprocess -import sys -from typing import Iterator, Tuple - - -def get_block_after_before_chunks( - after_block: int, - before_block: int, - n_workers: int, -) -> Iterator[Tuple[int, int]]: - n_blocks = before_block - after_block - remainder = n_blocks % n_workers - floor_chunk_size = n_blocks // n_workers - - last_before_block = None - - for worker_index in range(n_workers): - chunk_size = floor_chunk_size - - if worker_index < remainder: - chunk_size += 1 - - batch_after_block = ( - last_before_block if last_before_block is not None else after_block - ) - - batch_before_block = batch_after_block + chunk_size - yield batch_after_block, batch_before_block - last_before_block = batch_before_block - - -def backfill(after_block: int, before_block: int, n_workers: int): - if n_workers <= 0: - raise ValueError("Need at least one worker") - - for batch_after_block, batch_before_block in get_block_after_before_chunks( - after_block, - before_block, - n_workers, - ): - print(f"Backfilling {batch_after_block} to {batch_before_block}") - backfill_command = f"sh backfill.sh {batch_after_block} {batch_before_block}" - process = subprocess.Popen(backfill_command.split(), stdout=subprocess.PIPE) - output, _ = process.communicate() - print(output) - - -def main(): - after_block = int(sys.argv[1]) - before_block = int(sys.argv[2]) - n_workers = int(sys.argv[3]) - - backfill(after_block, before_block, n_workers) - - -if __name__ == "__main__": - main() diff --git a/backfill.sh b/backfill.sh deleted file mode 100644 index 5cc06af..0000000 --- a/backfill.sh +++ /dev/null @@ -1,6 +0,0 @@ -current_image=$(kubectl get deployment mev-inspect -o=jsonpath='{$.spec.template.spec.containers[:1].image}') - -helm template mev-inspect-backfill ./k8s/mev-inspect-backfill \ - --set image.repository=$current_image \ - --set command.startBlockNumber=$1 \ - --set command.endBlockNumber=$2 | kubectl apply -f - diff --git a/k8s/mev-inspect-backfill/.helmignore b/k8s/mev-inspect-backfill/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/k8s/mev-inspect-backfill/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/k8s/mev-inspect-backfill/Chart.yaml b/k8s/mev-inspect-backfill/Chart.yaml deleted file mode 100644 index 5ecfa0e..0000000 --- a/k8s/mev-inspect-backfill/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: mev-inspect-backfill -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" diff --git a/k8s/mev-inspect-backfill/templates/_helpers.tpl b/k8s/mev-inspect-backfill/templates/_helpers.tpl deleted file mode 100644 index e3650e6..0000000 --- a/k8s/mev-inspect-backfill/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "mev-inspect-backfill.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "mev-inspect-backfill.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "mev-inspect-backfill.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "mev-inspect-backfill.labels" -}} -helm.sh/chart: {{ include "mev-inspect-backfill.chart" . }} -{{ include "mev-inspect-backfill.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "mev-inspect-backfill.selectorLabels" -}} -app.kubernetes.io/name: {{ include "mev-inspect-backfill.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "mev-inspect-backfill.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "mev-inspect-backfill.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/k8s/mev-inspect-backfill/templates/job.yaml b/k8s/mev-inspect-backfill/templates/job.yaml deleted file mode 100644 index 6f0bc0b..0000000 --- a/k8s/mev-inspect-backfill/templates/job.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "mev-inspect-backfill.fullname" . }}-{{ randAlphaNum 5 | lower }} - labels: - {{- include "mev-inspect-backfill.labels" . | nindent 4 }} -spec: - completions: 1 - parallelism: 1 - ttlSecondsAfterFinished: 5 - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - run - - inspect-many-blocks - - {{ .Values.command.startBlockNumber | quote }} - - {{ .Values.command.endBlockNumber | quote }} - env: - - name: POSTGRES_HOST - valueFrom: - secretKeyRef: - name: mev-inspect-db-credentials - key: host - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: mev-inspect-db-credentials - key: username - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: mev-inspect-db-credentials - key: password - - name: TRACE_DB_HOST - valueFrom: - secretKeyRef: - name: trace-db-credentials - key: host - optional: true - - name: TRACE_DB_USER - valueFrom: - secretKeyRef: - name: trace-db-credentials - key: username - optional: true - - name: TRACE_DB_PASSWORD - valueFrom: - secretKeyRef: - name: trace-db-credentials - key: password - optional: true - - name: RPC_URL - valueFrom: - configMapKeyRef: - name: mev-inspect-rpc - key: url - restartPolicy: OnFailure diff --git a/k8s/mev-inspect-backfill/values.yaml b/k8s/mev-inspect-backfill/values.yaml deleted file mode 100644 index 3e45e4e..0000000 --- a/k8s/mev-inspect-backfill/values.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Default values for mev-inspect. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -image: - repository: mev-inspect-py - pullPolicy: IfNotPresent - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: {} - -tolerations: [] - -affinity: {}