Add configmap for RPC. Print latest block on loop

Tiltfile

@@ -1,6 +1,7 @@
 load('ext://helm_remote', 'helm_remote')
 load('ext://restart_process', 'docker_build_with_restart')
 load('ext://secret', 'secret_from_dict')
+load('ext://configmap', 'configmap_from_dict')
 
 helm_remote("postgresql",
             repo_name='bitnami',
@@ -8,6 +9,10 @@ helm_remote("postgresql",
             set=["postgresqlPassword=password", "postgresqlDatabase=mev_inspect"],
 )
 
+k8s_yaml(configmap_from_dict("mev-inspect-rpc", inputs = {
+    "url" : os.environ["RPC_URL"],
+}))
+
 k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = {
     "username" : "postgres",
     "password": "password",

k8s/app.yaml

@@ -31,6 +31,11 @@ spec:
               key: password
         - name: POSTGRES_HOST
           value: postgresql
+        - name: RPC_URL
+          valueFrom:
+            configMapKeyRef:
+              name: mev-inspect-rpc
+              key: url
         livenessProbe:
           exec:
             command:

mev_inspect/block.py

@@ -11,6 +11,10 @@ from mev_inspect.schemas.receipts import Receipt
 cache_directory = "./cache"
 
 
+def get_latest_block_number(w3: Web3) -> int:
+    return w3.eth.get_block("latest")["number"]
+
+
 def create_from_block_number(
     base_provider, w3: Web3, block_number: int, should_cache: bool
 ) -> Block:

mev_inspect/provider.py

@@ -0,0 +1,14 @@
+from web3 import Web3
+
+from mev_inspect.retry import http_retry_with_backoff_request_middleware
+
+
+def get_base_provider(rpc: str) -> Web3.HTTPProvider:
+    base_provider = Web3.HTTPProvider(rpc)
+    base_provider.middlewares.remove("http_retry_request")
+    base_provider.middlewares.add(
+        http_retry_with_backoff_request_middleware,
+        "http_retry_with_backoff",
+    )
+
+    return base_provider

run.py

@@ -1,9 +1,15 @@
 import logging
+import os
 import signal
 import time
 
+from web3 import Web3
 
-logging.basicConfig(filename="app.log", level=logging.DEBUG)
+from mev_inspect.block import get_latest_block_number
+from mev_inspect.provider import get_base_provider
+
+
+logging.basicConfig(filename="app.log", level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
@@ -28,11 +34,19 @@ class GracefulKiller:
 
 
 if __name__ == "__main__":
+    rpc = os.getenv("RPC_URL")
+    if rpc is None:
+        raise RuntimeError("Missing environment variable RPC_URL")
+
     logger.info("Starting...")
+
     killer = GracefulKiller()
+    base_provider = get_base_provider(rpc)
+    w3 = Web3(base_provider)
 
     while not killer.kill_now:
-        logger.info("Running...")
+        latest_block_number = get_latest_block_number(w3)
+        logger.info(f"Latest block: {latest_block_number}")
         time.sleep(5)
 
     logger.info("Stopping...")

scripts/inspect_commands.py

@@ -3,7 +3,7 @@ from web3 import Web3
 
 from mev_inspect.db import get_session
 from mev_inspect.inspect_block import inspect_block
-from mev_inspect.retry import http_retry_with_backoff_request_middleware
+from mev_inspect.provider import get_base_provider
 
 
 @click.group()
@@ -17,7 +17,7 @@ def cli():
 @click.option("--cache/--no-cache", default=True)
 def inspect_block_command(block_number: int, rpc: str, cache: bool):
     db_session = get_session()
-    base_provider = _get_base_provider(rpc)
+    base_provider = get_base_provider(rpc)
     w3 = Web3(base_provider)
 
     if not cache:
@@ -36,7 +36,7 @@ def inspect_many_blocks_command(
 ):
 
     db_session = get_session()
-    base_provider = _get_base_provider(rpc)
+    base_provider = get_base_provider(rpc)
     w3 = Web3(base_provider)
 
     if not cache:
@@ -61,16 +61,5 @@ def inspect_many_blocks_command(
         )
 
 
-def _get_base_provider(rpc: str) -> Web3.HTTPProvider:
-    base_provider = Web3.HTTPProvider(rpc)
-    base_provider.middlewares.remove("http_retry_request")
-    base_provider.middlewares.add(
-        http_retry_with_backoff_request_middleware,
-        "http_retry_with_backoff",
-    )
-
-    return base_provider
-
-
 if __name__ == "__main__":
     cli()

tilt_modules/configmap/README.md

@@ -0,0 +1,54 @@
+# Configmap
+
+Author: [Nick Santos](https://github.com/nicks)
+
+Helper functions for creating Kubernetes configmaps.
+
+## Functions
+
+### configmap_yaml
+
+```
+configmap_yaml(name: str, namespace: str = "", from_file: Union[str, List[str]] = None, watch: bool = True, from_env_file: str = None): Blob
+```
+
+Returns YAML for a config map generated from a file.
+
+* `from_file` ( str ) – equivalent to `kubectl create configmap --from-file`
+* `from_env_file` (str) - equivalent to `kubectl create configmap --from-env-file`
+* `watch` ( bool ) - auto-reload if the files change
+
+### configmap_create
+
+```
+configmap_create(name: str, namespace: str = "", from_file: Union[str, List[str]] = None, watch: bool = True, from_env_file: str = None)
+```
+
+Deploys a config map. Equivalent to
+
+```
+k8s_yaml(configmap_yaml('name', from_file=[...]))
+```
+
+### configmap_from_dict
+
+```
+configmap_from_dict(name: str, namespace: str = "", inputs: Dict[str, str]] = None): Blob
+```
+
+Returns YAML for a config map generated from a given dictionary. Nested dictionaries are not supported
+
+* `inputs` ( dict ) – equivalent to `kubectl create configmap --from-literal` for each key and value
+
+## Example Usage
+
+### For a Grafana config
+
+```
+load('ext://configmap', 'configmap_create')
+configmap_create('grafana-config', from_file=['grafana.ini=./grafana.ini'])
+```
+
+## Caveats
+
+- This extension doesn't do any validation to confirm that names or namespaces are valid.

tilt_modules/configmap/Tiltfile

@@ -0,0 +1,109 @@
+# -*- mode: Python -*-
+
+def configmap_yaml(name, namespace="", from_file=None, watch=True, from_env_file=None):
+  """Returns YAML for a generic configmap
+
+  Args:
+    name: The configmap name.
+    namespace: The namespace.
+    from_file: Use the from-file configmap generator. May be a string or a list of strings.
+       Example: ["grafana.ini=path/to/grafana.ini"]
+    watch: Reruns the Tiltfile and re-deploys automatically if the from-files change.
+       Defaults to true.
+    from_env_file: Use from-env-file configmap generator. Must be string.
+       Example: "./local.env"
+
+  Returns:
+    The configmap YAML as a blob
+  """
+
+  args = [
+    "kubectl",
+    "create",
+    "configmap",
+    name,
+  ]
+
+  if namespace:
+    args.extend(["-n", namespace])
+
+  generator = False
+
+  if from_file and from_env_file:
+    fail("Must specify either 'from_file' OR 'from_env_file'")
+
+  if from_file:
+    if type(from_file) == "string":
+      from_file = [from_file]
+
+    if type(from_file) == "list":
+      for f in from_file:
+        args.extend(["--from-file", f])
+        if watch:
+          l = f.split('=')
+          watch_file(l[len(l)-1])
+      generator = True
+    else:
+      fail("Bad from_file argument: %s" % from_file)
+  elif from_env_file:
+    if type(from_env_file) == "list":
+      fail("from_env_file only supports string as an input to prevent confusion with kubectl behavior of only loading the last item in a list")
+    elif type(from_env_file == "string"):
+      args.extend(["--from-env-file", from_env_file])
+      if watch:
+        watch_file(from_env_file)
+      generator = True
+
+  if not generator:
+    fail("No configmap generator specified")
+
+  args.extend(["-o=yaml", "--dry-run=client"])
+  return local(args, quiet=True)
+
+def configmap_from_dict(name, namespace="", inputs={}):
+    """Returns YAML for a generic configmap
+    Args:
+        name: The configmap name.
+        namespace: The namespace.
+        inputs: A dict of keys and values to use. Nesting is not supported
+    Returns:
+        The configmap YAML as a blob
+    """
+
+    args = [
+        "kubectl",
+        "create",
+        "configmap",
+        name,
+    ]
+
+    if namespace:
+        args.extend(["-n", namespace])
+
+    if type(inputs) != "dict":
+        fail("Bad argument to configmap_from_dict, inputs was not dict typed")
+
+    for k,v in inputs.items():
+        args.extend(["--from-literal", "%s=%s" % (k,v)])
+
+    args.extend(["-o=yaml", "--dry-run=client"])
+    return local(args, quiet=True)
+
+def configmap_create(name, namespace="", from_file=None, watch=True, from_env_file=None):
+  """Creates a configmap in the current Kubernetes cluster.
+
+  Generators:
+    - from_file: Wraps kubectl from-file behavior.
+    - from_env_file: Wraps kubectl from-env-file behavior.
+
+  Args:
+    name: The configmap name.
+    namespace: The namespace.
+    from_file: Use the from-file configmap generator. May be a string or a list of strings.
+       Example: ["grafana.ini=path/to/grafana.ini"]
+    watch: Reruns the Tiltfile and re-deploys automatically if the from-files change.
+       Defaults to true.
+    from_env_file: Use from-env-file configmap generator. Must be string.
+       Example: "./local.env"
+  """
+  k8s_yaml(configmap_yaml(name, namespace, from_file, watch, from_env_file))

tilt_modules/configmap/test/Tiltfile

@@ -0,0 +1,7 @@
+load('../Tiltfile', 'configmap_create', 'configmap_from_dict')
+
+configmap_create('job-config', from_file='my-job.ini=./my-job.ini')
+configmap_create('env-job-config', from_env_file='my-job.env')
+configmap_from_dict('from-dict-config', inputs={"hello":"world"})
+
+k8s_yaml('job.yaml')

tilt_modules/configmap/test/job.yaml

@@ -0,0 +1,29 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: configmap-verify
+spec:
+  backoffLimit: 1
+  template:
+    spec:
+      containers:
+      - name: configmap-env-verify
+        image: alpine
+        command: ["/bin/echo", "$(TEST_VAR)"]
+        env:
+          - name: TEST_VAR
+            valueFrom:
+              configMapKeyRef:
+                name: env-job-config
+                key: TEST_VAR
+      - name: configmap-verify
+        image: alpine
+        command: ["cat", "/etc/my-job/my-job.ini"]
+        volumeMounts:
+        - name: job-config
+          mountPath: /etc/my-job
+      restartPolicy: Never
+      volumes:
+      - name: job-config
+        configMap:
+          name: job-config

tilt_modules/configmap/test/my-job.env

@@ -0,0 +1 @@
+TEST_VAR="hello-env!"

tilt_modules/configmap/test/my-job.ini

@@ -0,0 +1 @@
+hello!

tilt_modules/configmap/test/test.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cd "$(dirname "$0")"
+
+set -ex
+tilt ci
+tilt down

tilt_modules/extensions.json

@@ -19,6 +19,11 @@
       "Name": "restart_process",
       "ExtensionRegistry": "https://github.com/tilt-dev/tilt-extensions",
       "TimeFetched": "2021-09-13T20:14:11.011803-04:00"
+    },
+    {
+      "Name": "configmap",
+      "ExtensionRegistry": "https://github.com/tilt-dev/tilt-extensions",
+      "TimeFetched": "2021-09-13T20:58:06.169124-04:00"
     }
   ]
 }