Merge pull request #218 from flashbots/remove-backfill
Remove old backfill code
This commit is contained in:
Luke Van Seters
GitHub
commit
a6e76bfd10
57
backfill.py
57
backfill.py
@ -1,57 +0,0 @@
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Iterator, Tuple
|
||||
|
||||
|
||||
def get_block_after_before_chunks(
|
||||
after_block: int,
|
||||
before_block: int,
|
||||
n_workers: int,
|
||||
) -> Iterator[Tuple[int, int]]:
|
||||
n_blocks = before_block - after_block
|
||||
remainder = n_blocks % n_workers
|
||||
floor_chunk_size = n_blocks // n_workers
|
||||
|
||||
last_before_block = None
|
||||
|
||||
for worker_index in range(n_workers):
|
||||
chunk_size = floor_chunk_size
|
||||
|
||||
if worker_index < remainder:
|
||||
chunk_size += 1
|
||||
|
||||
batch_after_block = (
|
||||
last_before_block if last_before_block is not None else after_block
|
||||
)
|
||||
|
||||
batch_before_block = batch_after_block + chunk_size
|
||||
yield batch_after_block, batch_before_block
|
||||
last_before_block = batch_before_block
|
||||
|
||||
|
||||
def backfill(after_block: int, before_block: int, n_workers: int):
|
||||
if n_workers <= 0:
|
||||
raise ValueError("Need at least one worker")
|
||||
|
||||
for batch_after_block, batch_before_block in get_block_after_before_chunks(
|
||||
after_block,
|
||||
before_block,
|
||||
n_workers,
|
||||
):
|
||||
print(f"Backfilling {batch_after_block} to {batch_before_block}")
|
||||
backfill_command = f"sh backfill.sh {batch_after_block} {batch_before_block}"
|
||||
process = subprocess.Popen(backfill_command.split(), stdout=subprocess.PIPE)
|
||||
output, _ = process.communicate()
|
||||
print(output)
|
||||
|
||||
|
||||
def main():
|
||||
after_block = int(sys.argv[1])
|
||||
before_block = int(sys.argv[2])
|
||||
n_workers = int(sys.argv[3])
|
||||
|
||||
backfill(after_block, before_block, n_workers)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,6 +0,0 @@
|
||||
current_image=$(kubectl get deployment mev-inspect -o=jsonpath='{$.spec.template.spec.containers[:1].image}')
|
||||
|
||||
helm template mev-inspect-backfill ./k8s/mev-inspect-backfill \
|
||||
--set image.repository=$current_image \
|
||||
--set command.startBlockNumber=$1 \
|
||||
--set command.endBlockNumber=$2 | kubectl apply -f -
|
||||
@ -1,23 +0,0 @@
|
||||
# Patterns to ignore when building packages.
|
||||
# This supports shell glob matching, relative path matching, and
|
||||
# negation (prefixed with !). Only one pattern per line.
|
||||
.DS_Store
|
||||
# Common VCS dirs
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
# Common backup files
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*.orig
|
||||
*~
|
||||
# Various IDEs
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
||||
.vscode/
|
||||
@ -1,24 +0,0 @@
|
||||
apiVersion: v2
|
||||
name: mev-inspect-backfill
|
||||
description: A Helm chart for Kubernetes
|
||||
|
||||
# A chart can be either an 'application' or a 'library' chart.
|
||||
#
|
||||
# Application charts are a collection of templates that can be packaged into versioned archives
|
||||
# to be deployed.
|
||||
#
|
||||
# Library charts provide useful utilities or functions for the chart developer. They're included as
|
||||
# a dependency of application charts to inject those utilities and functions into the rendering
|
||||
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
|
||||
type: application
|
||||
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: 0.1.0
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
# follow Semantic Versioning. They should reflect the version the application is using.
|
||||
# It is recommended to use it with quotes.
|
||||
appVersion: "1.16.0"
|
||||
@ -1,62 +0,0 @@
|
||||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "mev-inspect-backfill.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "mev-inspect-backfill.fullname" -}}
|
||||
{{- if .Values.fullnameOverride }}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "mev-inspect-backfill.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "mev-inspect-backfill.labels" -}}
|
||||
helm.sh/chart: {{ include "mev-inspect-backfill.chart" . }}
|
||||
{{ include "mev-inspect-backfill.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels
|
||||
*/}}
|
||||
{{- define "mev-inspect-backfill.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "mev-inspect-backfill.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create the name of the service account to use
|
||||
*/}}
|
||||
{{- define "mev-inspect-backfill.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount.create }}
|
||||
{{- default (include "mev-inspect-backfill.fullname" .) .Values.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@ -1,68 +0,0 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "mev-inspect-backfill.fullname" . }}-{{ randAlphaNum 5 | lower }}
|
||||
labels:
|
||||
{{- include "mev-inspect-backfill.labels" . | nindent 4 }}
|
||||
spec:
|
||||
completions: 1
|
||||
parallelism: 1
|
||||
ttlSecondsAfterFinished: 5
|
||||
template:
|
||||
metadata:
|
||||
{{- with .Values.podAnnotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
containers:
|
||||
- name: {{ .Chart.Name }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.securityContext | nindent 12 }}
|
||||
image: "{{ .Values.image.repository }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
args:
|
||||
- run
|
||||
- inspect-many-blocks
|
||||
- {{ .Values.command.startBlockNumber | quote }}
|
||||
- {{ .Values.command.endBlockNumber | quote }}
|
||||
env:
|
||||
- name: POSTGRES_HOST
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mev-inspect-db-credentials
|
||||
key: host
|
||||
- name: POSTGRES_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mev-inspect-db-credentials
|
||||
key: username
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mev-inspect-db-credentials
|
||||
key: password
|
||||
- name: TRACE_DB_HOST
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trace-db-credentials
|
||||
key: host
|
||||
optional: true
|
||||
- name: TRACE_DB_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trace-db-credentials
|
||||
key: username
|
||||
optional: true
|
||||
- name: TRACE_DB_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trace-db-credentials
|
||||
key: password
|
||||
optional: true
|
||||
- name: RPC_URL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: mev-inspect-rpc
|
||||
key: url
|
||||
restartPolicy: OnFailure
|
||||
@ -1,42 +0,0 @@
|
||||
# Default values for mev-inspect.
|
||||
# This is a YAML-formatted file.
|
||||
# Declare variables to be passed into your templates.
|
||||
|
||||
image:
|
||||
repository: mev-inspect-py
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
imagePullSecrets: []
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
podAnnotations: {}
|
||||
|
||||
podSecurityContext: {}
|
||||
# fsGroup: 2000
|
||||
|
||||
securityContext: {}
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# readOnlyRootFilesystem: true
|
||||
# runAsNonRoot: true
|
||||
# runAsUser: 1000
|
||||
|
||||
resources: {}
|
||||
# We usually recommend not to specify default resources and to leave this as a conscious
|
||||
# choice for the user. This also increases chances charts run on environments with little
|
||||
# resources, such as Minikube. If you do want to specify resources, uncomment the following
|
||||
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
|
||||
nodeSelector: {}
|
||||
|
||||
tolerations: []
|
||||
|
||||
affinity: {}
|
||||
Loading…
x
Reference in New Issue
Block a user