From a8c1728e357032fd2bdf238a4858ed469c67ef55 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Fri, 19 Nov 2021 08:20:40 -0500 Subject: [PATCH 1/7] Save progress --- Dockerfile | 2 +- Tiltfile | 6 +- k8s/mev-inspect-backfill/templates/job.yaml | 3 +- k8s/mev-inspect-prices/.helmignore | 23 +++++++ k8s/mev-inspect-prices/Chart.yaml | 24 +++++++ k8s/mev-inspect-prices/templates/_helpers.tpl | 62 +++++++++++++++++++ k8s/mev-inspect-prices/templates/cronjob.yaml | 36 +++++++++++ k8s/mev-inspect-prices/values.yaml | 7 +++ k8s/mev-inspect/templates/deployment.yaml | 1 + 9 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 k8s/mev-inspect-prices/.helmignore create mode 100644 k8s/mev-inspect-prices/Chart.yaml create mode 100644 k8s/mev-inspect-prices/templates/_helpers.tpl create mode 100644 k8s/mev-inspect-prices/templates/cronjob.yaml create mode 100644 k8s/mev-inspect-prices/values.yaml diff --git a/Dockerfile b/Dockerfile index 5801d11..5a950e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,4 @@ COPY . /app # easter eggs 😝 RUN echo "PS1='🕵️:\[\033[1;36m\]\h \[\033[1;34m\]\W\[\033[0;35m\]\[\033[1;36m\]$ \[\033[0m\]'" >> ~/.bashrc -ENTRYPOINT [ "/app/entrypoint.sh"] +ENTRYPOINT [ "poetry"] diff --git a/Tiltfile b/Tiltfile index 9f8ec7d..585d18f 100644 --- a/Tiltfile +++ b/Tiltfile @@ -31,16 +31,20 @@ k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = { # })) docker_build_with_restart("mev-inspect-py", ".", - entrypoint="/app/entrypoint.sh", + entrypoint="poetry", live_update=[ sync(".", "/app"), run("cd /app && poetry install", trigger="./pyproject.toml"), ], + platform='linux/arm64', ) k8s_yaml(helm('./k8s/mev-inspect', name='mev-inspect')) k8s_resource(workload="mev-inspect", resource_deps=["postgresql-postgresql"]) +k8s_yaml(helm('./k8s/mev-inspect-prices', name='mev-inspect-prices')) +k8s_resource(workload="mev-inspect-prices", resource_deps=["postgresql-postgresql"]) + local_resource( 'pg-port-forward', serve_cmd='kubectl port-forward --namespace default svc/postgresql 5432:5432', diff --git a/k8s/mev-inspect-backfill/templates/job.yaml b/k8s/mev-inspect-backfill/templates/job.yaml index 35f07b0..6f0bc0b 100644 --- a/k8s/mev-inspect-backfill/templates/job.yaml +++ b/k8s/mev-inspect-backfill/templates/job.yaml @@ -21,8 +21,7 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - command: - - poetry + args: - run - inspect-many-blocks - {{ .Values.command.startBlockNumber | quote }} diff --git a/k8s/mev-inspect-prices/.helmignore b/k8s/mev-inspect-prices/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/k8s/mev-inspect-prices/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s/mev-inspect-prices/Chart.yaml b/k8s/mev-inspect-prices/Chart.yaml new file mode 100644 index 0000000..5083c60 --- /dev/null +++ b/k8s/mev-inspect-prices/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: mev-inspect-prices +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/k8s/mev-inspect-prices/templates/_helpers.tpl b/k8s/mev-inspect-prices/templates/_helpers.tpl new file mode 100644 index 0000000..1d8a659 --- /dev/null +++ b/k8s/mev-inspect-prices/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "mev-inspect-prices.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "mev-inspect-prices.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "mev-inspect-prices.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "mev-inspect-prices.labels" -}} +helm.sh/chart: {{ include "mev-inspect-prices.chart" . }} +{{ include "mev-inspect-prices.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "mev-inspect-prices.selectorLabels" -}} +app.kubernetes.io/name: {{ include "mev-inspect-prices.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "mev-inspect-prices.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "mev-inspect-prices.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/k8s/mev-inspect-prices/templates/cronjob.yaml b/k8s/mev-inspect-prices/templates/cronjob.yaml new file mode 100644 index 0000000..4c982bc --- /dev/null +++ b/k8s/mev-inspect-prices/templates/cronjob.yaml @@ -0,0 +1,36 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "mev-inspect-prices.fullname" . }} +spec: + completions: 1 + ttlSecondsAfterFinished: 5 + schedule: "*/1 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - run + - fetch-all-prices + env: + - name: POSTGRES_HOST + valueFrom: + secretKeyRef: + name: mev-inspect-db-credentials + key: host + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: mev-inspect-db-credentials + key: username + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: mev-inspect-db-credentials + key: password + restartPolicy: OnFailure diff --git a/k8s/mev-inspect-prices/values.yaml b/k8s/mev-inspect-prices/values.yaml new file mode 100644 index 0000000..77238c0 --- /dev/null +++ b/k8s/mev-inspect-prices/values.yaml @@ -0,0 +1,7 @@ +image: + repository: mev-inspect-py + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" diff --git a/k8s/mev-inspect/templates/deployment.yaml b/k8s/mev-inspect/templates/deployment.yaml index 9aa4a50..ec2045c 100644 --- a/k8s/mev-inspect/templates/deployment.yaml +++ b/k8s/mev-inspect/templates/deployment.yaml @@ -30,6 +30,7 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + args: ["run", "python", "loop.py"] livenessProbe: exec: command: From 34aca861ccfb37a81801aa577b73a54be2d048d6 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 22 Nov 2021 13:03:50 -0500 Subject: [PATCH 2/7] Use poetry directly instead of entrypoint script --- Dockerfile | 3 ++- Tiltfile | 4 +--- entrypoint.sh | 3 --- 3 files changed, 3 insertions(+), 7 deletions(-) delete mode 100755 entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 5a950e8..569df05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,4 +18,5 @@ COPY . /app # easter eggs 😝 RUN echo "PS1='🕵️:\[\033[1;36m\]\h \[\033[1;34m\]\W\[\033[0;35m\]\[\033[1;36m\]$ \[\033[0m\]'" >> ~/.bashrc -ENTRYPOINT [ "poetry"] +ENTRYPOINT [ "poetry" ] +CMD [ "run", "python", "loop.py" ] diff --git a/Tiltfile b/Tiltfile index 585d18f..a828272 100644 --- a/Tiltfile +++ b/Tiltfile @@ -30,14 +30,12 @@ k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = { # "host": "trace-db-postgresql", # })) -docker_build_with_restart("mev-inspect-py", ".", - entrypoint="poetry", +docker_build("mev-inspect-py", ".", live_update=[ sync(".", "/app"), run("cd /app && poetry install", trigger="./pyproject.toml"), ], - platform='linux/arm64', ) k8s_yaml(helm('./k8s/mev-inspect', name='mev-inspect')) k8s_resource(workload="mev-inspect", resource_deps=["postgresql-postgresql"]) diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100755 index 54f1f8a..0000000 --- a/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -python loop.py From c83577b04c4b4acb6493e03513681b2e29967018 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 22 Nov 2021 13:06:30 -0500 Subject: [PATCH 3/7] Remove restart --- Tiltfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Tiltfile b/Tiltfile index a828272..1be021c 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,5 +1,4 @@ load("ext://helm_remote", "helm_remote") -load("ext://restart_process", "docker_build_with_restart") load("ext://secret", "secret_from_dict") load("ext://configmap", "configmap_from_dict") From 391314b9d6ee618dfba8e9125a4b510c828aca2c Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 22 Nov 2021 13:51:35 -0500 Subject: [PATCH 4/7] Limit successful history instead of ttl --- k8s/mev-inspect-prices/templates/cronjob.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/k8s/mev-inspect-prices/templates/cronjob.yaml b/k8s/mev-inspect-prices/templates/cronjob.yaml index 4c982bc..1c1379b 100644 --- a/k8s/mev-inspect-prices/templates/cronjob.yaml +++ b/k8s/mev-inspect-prices/templates/cronjob.yaml @@ -3,9 +3,8 @@ kind: CronJob metadata: name: {{ include "mev-inspect-prices.fullname" . }} spec: - completions: 1 - ttlSecondsAfterFinished: 5 schedule: "*/1 * * * *" + successfulJobsHistoryLimit: 0 jobTemplate: spec: template: @@ -33,4 +32,4 @@ spec: secretKeyRef: name: mev-inspect-db-credentials key: password - restartPolicy: OnFailure + restartPolicy: Never From 9a076a6b4cf576dd5c619e45423d99aa3b4d4601 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 22 Nov 2021 13:52:27 -0500 Subject: [PATCH 5/7] Don't run prices by default --- Tiltfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tiltfile b/Tiltfile index 1be021c..afc0d92 100644 --- a/Tiltfile +++ b/Tiltfile @@ -39,8 +39,9 @@ docker_build("mev-inspect-py", ".", k8s_yaml(helm('./k8s/mev-inspect', name='mev-inspect')) k8s_resource(workload="mev-inspect", resource_deps=["postgresql-postgresql"]) -k8s_yaml(helm('./k8s/mev-inspect-prices', name='mev-inspect-prices')) -k8s_resource(workload="mev-inspect-prices", resource_deps=["postgresql-postgresql"]) +# uncomment to enable price monitor +# k8s_yaml(helm('./k8s/mev-inspect-prices', name='mev-inspect-prices')) +# k8s_resource(workload="mev-inspect-prices", resource_deps=["postgresql-postgresql"]) local_resource( 'pg-port-forward', From 018fb8c73bdfe2745a6e4caae58c0e64734397ed Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 22 Nov 2021 13:54:12 -0500 Subject: [PATCH 6/7] Run hourly --- k8s/mev-inspect-prices/templates/cronjob.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/mev-inspect-prices/templates/cronjob.yaml b/k8s/mev-inspect-prices/templates/cronjob.yaml index 1c1379b..900d0ab 100644 --- a/k8s/mev-inspect-prices/templates/cronjob.yaml +++ b/k8s/mev-inspect-prices/templates/cronjob.yaml @@ -3,7 +3,7 @@ kind: CronJob metadata: name: {{ include "mev-inspect-prices.fullname" . }} spec: - schedule: "*/1 * * * *" + schedule: "0 */1 * * *" successfulJobsHistoryLimit: 0 jobTemplate: spec: From ed94e717153cbe5a735967b2bc4a48d01694f8c0 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Tue, 30 Nov 2021 12:54:07 -0500 Subject: [PATCH 7/7] Fix timestamp writing in blocks --- mev_inspect/crud/blocks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mev_inspect/crud/blocks.py b/mev_inspect/crud/blocks.py index 3639ba8..90ead13 100644 --- a/mev_inspect/crud/blocks.py +++ b/mev_inspect/crud/blocks.py @@ -1,3 +1,5 @@ +from datetime import datetime + from mev_inspect.schemas.blocks import Block @@ -20,7 +22,7 @@ def write_block( "INSERT INTO blocks (block_number, block_timestamp) VALUES (:block_number, :block_timestamp)", params={ "block_number": block.block_number, - "block_timestamp": block.block_timestamp, + "block_timestamp": datetime.fromtimestamp(block.block_timestamp), }, ) db_session.commit()