Kubernetes integration

Tiltfile

@@ -72,3 +72,7 @@ local_resource(
     serve_cmd='kubectl port-forward --namespace default svc/postgresql 5432:5432',
     resource_deps=["postgresql-postgresql"]
 )
+
+k8s_yaml(configmap_from_dict("mev-inspect-s3", inputs = {
+    "uri" : "https://s3.us-east-1.amazonaws.com/mybucket/"
+}))

cli.py

@@ -13,6 +13,7 @@ from mev_inspect.inspector import MEVInspector
 from mev_inspect.prices import fetch_prices, fetch_prices_range
 
 RPC_URL_ENV = "RPC_URL"
+S3_URI_ENV = "S3_URI"
 
 logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -134,7 +135,7 @@ def fetch_range(after: datetime, before: datetime):
 
 @cli.command()
 @click.argument("block_number", type=int)
-@click.argument("uri", type=str)
+@click.option("--uri", default=lambda: os.environ.get(S3_URI_ENV, ""))
 def s3_export_command(block_number: int, uri: str):
     inspect_db_session = get_inspect_session()
 
@@ -147,12 +148,12 @@ def s3_export_command(block_number: int, uri: str):
 @cli.command()
 @click.argument("after_block", type=int)
 @click.argument("before_block", type=int)
-@click.argument("base_uri", type=str)
+@click.option("--uri", default=lambda: os.environ.get(S3_URI_ENV, ""))
-def s3_export_many_command(after_block: int, before_block: int, base_uri: str):
+def s3_export_many_command(after_block: int, before_block: int, uri: str):
     inspect_db_session = get_inspect_session()
 
     logger.info(f"Exporting blocks {after_block} to {before_block}")
-    s3_export_many(inspect_db_session, after_block, before_block, base_uri)
+    s3_export_many(inspect_db_session, after_block, before_block, uri)
 
     return None
 
@@ -161,5 +162,9 @@ def get_rpc_url() -> str:
     return os.environ["RPC_URL"]
 
 
+def get_s3_urii() -> str:
+    return os.environ["S3_URI"]
+
+
 if __name__ == "__main__":
     cli()

k8s/mev-inspect/templates/deployment.yaml

@@ -91,6 +91,11 @@ spec:
                 name: mev-inspect-listener-healthcheck
                 key: url
                 optional: true
+          - name: S3_URI
+            valueFrom:
+              configMapKeyRef:
+                name: mev-inspect-s3
+                key: uri
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

mev_inspect/export.py

@@ -34,7 +34,7 @@ def s3_export_many(
     db_session,
     after_block: int,
     before_block: int,
-    base_uri: str,
+    uri: str,
 ) -> None:
     """Export block range to S3"""
 
@@ -46,7 +46,7 @@ def s3_export_many(
 
             if block_number > latest_s3_block:
 
-                uri = base_uri + f"/{block_number}"
+                uri += f"/{block_number}"
                 db_session.execute(
                     """
                     SELECT * FROM aws_s3.query_export_to_s3(