From e6793ee053be097ba265e798490019e99739f571 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Mon, 13 Sep 2021 20:59:52 -0400 Subject: [PATCH] Add configmap for RPC. Print latest block on loop --- Tiltfile | 5 ++ k8s/app.yaml | 5 ++ mev_inspect/block.py | 4 + mev_inspect/provider.py | 14 ++++ run.py | 18 +++- scripts/inspect_commands.py | 17 +--- tilt_modules/configmap/README.md | 54 ++++++++++++ tilt_modules/configmap/Tiltfile | 109 +++++++++++++++++++++++++ tilt_modules/configmap/test/Tiltfile | 7 ++ tilt_modules/configmap/test/job.yaml | 29 +++++++ tilt_modules/configmap/test/my-job.env | 1 + tilt_modules/configmap/test/my-job.ini | 1 + tilt_modules/configmap/test/test.sh | 7 ++ tilt_modules/extensions.json | 5 ++ 14 files changed, 260 insertions(+), 16 deletions(-) create mode 100644 mev_inspect/provider.py create mode 100644 tilt_modules/configmap/README.md create mode 100644 tilt_modules/configmap/Tiltfile create mode 100644 tilt_modules/configmap/test/Tiltfile create mode 100644 tilt_modules/configmap/test/job.yaml create mode 100644 tilt_modules/configmap/test/my-job.env create mode 100644 tilt_modules/configmap/test/my-job.ini create mode 100755 tilt_modules/configmap/test/test.sh diff --git a/Tiltfile b/Tiltfile index abc17cb..10714ab 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,6 +1,7 @@ load('ext://helm_remote', 'helm_remote') load('ext://restart_process', 'docker_build_with_restart') load('ext://secret', 'secret_from_dict') +load('ext://configmap', 'configmap_from_dict') helm_remote("postgresql", repo_name='bitnami', @@ -8,6 +9,10 @@ helm_remote("postgresql", set=["postgresqlPassword=password", "postgresqlDatabase=mev_inspect"], ) +k8s_yaml(configmap_from_dict("mev-inspect-rpc", inputs = { + "url" : os.environ["RPC_URL"], +})) + k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = { "username" : "postgres", "password": "password", diff --git a/k8s/app.yaml b/k8s/app.yaml index da45f2e..d5cabc1 100644 --- a/k8s/app.yaml +++ b/k8s/app.yaml @@ -31,6 +31,11 @@ spec: key: password - name: POSTGRES_HOST value: postgresql + - name: RPC_URL + valueFrom: + configMapKeyRef: + name: mev-inspect-rpc + key: url livenessProbe: exec: command: diff --git a/mev_inspect/block.py b/mev_inspect/block.py index dca522d..c05b16c 100644 --- a/mev_inspect/block.py +++ b/mev_inspect/block.py @@ -11,6 +11,10 @@ from mev_inspect.schemas.receipts import Receipt cache_directory = "./cache" +def get_latest_block_number(w3: Web3) -> int: + return w3.eth.get_block("latest")["number"] + + def create_from_block_number( base_provider, w3: Web3, block_number: int, should_cache: bool ) -> Block: diff --git a/mev_inspect/provider.py b/mev_inspect/provider.py new file mode 100644 index 0000000..1bdc68d --- /dev/null +++ b/mev_inspect/provider.py @@ -0,0 +1,14 @@ +from web3 import Web3 + +from mev_inspect.retry import http_retry_with_backoff_request_middleware + + +def get_base_provider(rpc: str) -> Web3.HTTPProvider: + base_provider = Web3.HTTPProvider(rpc) + base_provider.middlewares.remove("http_retry_request") + base_provider.middlewares.add( + http_retry_with_backoff_request_middleware, + "http_retry_with_backoff", + ) + + return base_provider diff --git a/run.py b/run.py index 5938f8e..7a93d5e 100644 --- a/run.py +++ b/run.py @@ -1,9 +1,15 @@ import logging +import os import signal import time +from web3 import Web3 -logging.basicConfig(filename="app.log", level=logging.DEBUG) +from mev_inspect.block import get_latest_block_number +from mev_inspect.provider import get_base_provider + + +logging.basicConfig(filename="app.log", level=logging.INFO) logger = logging.getLogger(__name__) @@ -28,11 +34,19 @@ class GracefulKiller: if __name__ == "__main__": + rpc = os.getenv("RPC_URL") + if rpc is None: + raise RuntimeError("Missing environment variable RPC_URL") + logger.info("Starting...") + killer = GracefulKiller() + base_provider = get_base_provider(rpc) + w3 = Web3(base_provider) while not killer.kill_now: - logger.info("Running...") + latest_block_number = get_latest_block_number(w3) + logger.info(f"Latest block: {latest_block_number}") time.sleep(5) logger.info("Stopping...") diff --git a/scripts/inspect_commands.py b/scripts/inspect_commands.py index ec38408..ceb2138 100644 --- a/scripts/inspect_commands.py +++ b/scripts/inspect_commands.py @@ -3,7 +3,7 @@ from web3 import Web3 from mev_inspect.db import get_session from mev_inspect.inspect_block import inspect_block -from mev_inspect.retry import http_retry_with_backoff_request_middleware +from mev_inspect.provider import get_base_provider @click.group() @@ -17,7 +17,7 @@ def cli(): @click.option("--cache/--no-cache", default=True) def inspect_block_command(block_number: int, rpc: str, cache: bool): db_session = get_session() - base_provider = _get_base_provider(rpc) + base_provider = get_base_provider(rpc) w3 = Web3(base_provider) if not cache: @@ -36,7 +36,7 @@ def inspect_many_blocks_command( ): db_session = get_session() - base_provider = _get_base_provider(rpc) + base_provider = get_base_provider(rpc) w3 = Web3(base_provider) if not cache: @@ -61,16 +61,5 @@ def inspect_many_blocks_command( ) -def _get_base_provider(rpc: str) -> Web3.HTTPProvider: - base_provider = Web3.HTTPProvider(rpc) - base_provider.middlewares.remove("http_retry_request") - base_provider.middlewares.add( - http_retry_with_backoff_request_middleware, - "http_retry_with_backoff", - ) - - return base_provider - - if __name__ == "__main__": cli() diff --git a/tilt_modules/configmap/README.md b/tilt_modules/configmap/README.md new file mode 100644 index 0000000..bbed5eb --- /dev/null +++ b/tilt_modules/configmap/README.md @@ -0,0 +1,54 @@ +# Configmap + +Author: [Nick Santos](https://github.com/nicks) + +Helper functions for creating Kubernetes configmaps. + +## Functions + +### configmap_yaml + +``` +configmap_yaml(name: str, namespace: str = "", from_file: Union[str, List[str]] = None, watch: bool = True, from_env_file: str = None): Blob +``` + +Returns YAML for a config map generated from a file. + +* `from_file` ( str ) – equivalent to `kubectl create configmap --from-file` +* `from_env_file` (str) - equivalent to `kubectl create configmap --from-env-file` +* `watch` ( bool ) - auto-reload if the files change + +### configmap_create + +``` +configmap_create(name: str, namespace: str = "", from_file: Union[str, List[str]] = None, watch: bool = True, from_env_file: str = None) +``` + +Deploys a config map. Equivalent to + +``` +k8s_yaml(configmap_yaml('name', from_file=[...])) +``` + +### configmap_from_dict + +``` +configmap_from_dict(name: str, namespace: str = "", inputs: Dict[str, str]] = None): Blob +``` + +Returns YAML for a config map generated from a given dictionary. Nested dictionaries are not supported + +* `inputs` ( dict ) – equivalent to `kubectl create configmap --from-literal` for each key and value + +## Example Usage + +### For a Grafana config + +``` +load('ext://configmap', 'configmap_create') +configmap_create('grafana-config', from_file=['grafana.ini=./grafana.ini']) +``` + +## Caveats + +- This extension doesn't do any validation to confirm that names or namespaces are valid. diff --git a/tilt_modules/configmap/Tiltfile b/tilt_modules/configmap/Tiltfile new file mode 100644 index 0000000..71bc9cd --- /dev/null +++ b/tilt_modules/configmap/Tiltfile @@ -0,0 +1,109 @@ +# -*- mode: Python -*- + +def configmap_yaml(name, namespace="", from_file=None, watch=True, from_env_file=None): + """Returns YAML for a generic configmap + + Args: + name: The configmap name. + namespace: The namespace. + from_file: Use the from-file configmap generator. May be a string or a list of strings. + Example: ["grafana.ini=path/to/grafana.ini"] + watch: Reruns the Tiltfile and re-deploys automatically if the from-files change. + Defaults to true. + from_env_file: Use from-env-file configmap generator. Must be string. + Example: "./local.env" + + Returns: + The configmap YAML as a blob + """ + + args = [ + "kubectl", + "create", + "configmap", + name, + ] + + if namespace: + args.extend(["-n", namespace]) + + generator = False + + if from_file and from_env_file: + fail("Must specify either 'from_file' OR 'from_env_file'") + + if from_file: + if type(from_file) == "string": + from_file = [from_file] + + if type(from_file) == "list": + for f in from_file: + args.extend(["--from-file", f]) + if watch: + l = f.split('=') + watch_file(l[len(l)-1]) + generator = True + else: + fail("Bad from_file argument: %s" % from_file) + elif from_env_file: + if type(from_env_file) == "list": + fail("from_env_file only supports string as an input to prevent confusion with kubectl behavior of only loading the last item in a list") + elif type(from_env_file == "string"): + args.extend(["--from-env-file", from_env_file]) + if watch: + watch_file(from_env_file) + generator = True + + if not generator: + fail("No configmap generator specified") + + args.extend(["-o=yaml", "--dry-run=client"]) + return local(args, quiet=True) + +def configmap_from_dict(name, namespace="", inputs={}): + """Returns YAML for a generic configmap + Args: + name: The configmap name. + namespace: The namespace. + inputs: A dict of keys and values to use. Nesting is not supported + Returns: + The configmap YAML as a blob + """ + + args = [ + "kubectl", + "create", + "configmap", + name, + ] + + if namespace: + args.extend(["-n", namespace]) + + if type(inputs) != "dict": + fail("Bad argument to configmap_from_dict, inputs was not dict typed") + + for k,v in inputs.items(): + args.extend(["--from-literal", "%s=%s" % (k,v)]) + + args.extend(["-o=yaml", "--dry-run=client"]) + return local(args, quiet=True) + +def configmap_create(name, namespace="", from_file=None, watch=True, from_env_file=None): + """Creates a configmap in the current Kubernetes cluster. + + Generators: + - from_file: Wraps kubectl from-file behavior. + - from_env_file: Wraps kubectl from-env-file behavior. + + Args: + name: The configmap name. + namespace: The namespace. + from_file: Use the from-file configmap generator. May be a string or a list of strings. + Example: ["grafana.ini=path/to/grafana.ini"] + watch: Reruns the Tiltfile and re-deploys automatically if the from-files change. + Defaults to true. + from_env_file: Use from-env-file configmap generator. Must be string. + Example: "./local.env" + """ + k8s_yaml(configmap_yaml(name, namespace, from_file, watch, from_env_file)) diff --git a/tilt_modules/configmap/test/Tiltfile b/tilt_modules/configmap/test/Tiltfile new file mode 100644 index 0000000..1e7b9dc --- /dev/null +++ b/tilt_modules/configmap/test/Tiltfile @@ -0,0 +1,7 @@ +load('../Tiltfile', 'configmap_create', 'configmap_from_dict') + +configmap_create('job-config', from_file='my-job.ini=./my-job.ini') +configmap_create('env-job-config', from_env_file='my-job.env') +configmap_from_dict('from-dict-config', inputs={"hello":"world"}) + +k8s_yaml('job.yaml') \ No newline at end of file diff --git a/tilt_modules/configmap/test/job.yaml b/tilt_modules/configmap/test/job.yaml new file mode 100644 index 0000000..1d43252 --- /dev/null +++ b/tilt_modules/configmap/test/job.yaml @@ -0,0 +1,29 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: configmap-verify +spec: + backoffLimit: 1 + template: + spec: + containers: + - name: configmap-env-verify + image: alpine + command: ["/bin/echo", "$(TEST_VAR)"] + env: + - name: TEST_VAR + valueFrom: + configMapKeyRef: + name: env-job-config + key: TEST_VAR + - name: configmap-verify + image: alpine + command: ["cat", "/etc/my-job/my-job.ini"] + volumeMounts: + - name: job-config + mountPath: /etc/my-job + restartPolicy: Never + volumes: + - name: job-config + configMap: + name: job-config diff --git a/tilt_modules/configmap/test/my-job.env b/tilt_modules/configmap/test/my-job.env new file mode 100644 index 0000000..c3b8b31 --- /dev/null +++ b/tilt_modules/configmap/test/my-job.env @@ -0,0 +1 @@ +TEST_VAR="hello-env!" \ No newline at end of file diff --git a/tilt_modules/configmap/test/my-job.ini b/tilt_modules/configmap/test/my-job.ini new file mode 100644 index 0000000..3462721 --- /dev/null +++ b/tilt_modules/configmap/test/my-job.ini @@ -0,0 +1 @@ +hello! \ No newline at end of file diff --git a/tilt_modules/configmap/test/test.sh b/tilt_modules/configmap/test/test.sh new file mode 100755 index 0000000..a49e009 --- /dev/null +++ b/tilt_modules/configmap/test/test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cd "$(dirname "$0")" + +set -ex +tilt ci +tilt down diff --git a/tilt_modules/extensions.json b/tilt_modules/extensions.json index 4d58ca7..df6e050 100644 --- a/tilt_modules/extensions.json +++ b/tilt_modules/extensions.json @@ -19,6 +19,11 @@ "Name": "restart_process", "ExtensionRegistry": "https://github.com/tilt-dev/tilt-extensions", "TimeFetched": "2021-09-13T20:14:11.011803-04:00" + }, + { + "Name": "configmap", + "ExtensionRegistry": "https://github.com/tilt-dev/tilt-extensions", + "TimeFetched": "2021-09-13T20:58:06.169124-04:00" } ] } \ No newline at end of file