From f696bb72f4df1b5f7a22100595e41bd894de5c46 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 3 Feb 2022 15:39:54 -0500 Subject: [PATCH 01/17] Add localstack --- Tiltfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Tiltfile b/Tiltfile index 7dc1ec1..445399d 100644 --- a/Tiltfile +++ b/Tiltfile @@ -2,6 +2,11 @@ load("ext://helm_remote", "helm_remote") load("ext://secret", "secret_from_dict") load("ext://configmap", "configmap_from_dict") +helm_remote("localstack", + repo_name="localstack-charts", + repo_url="https://localstack.github.io/helm-charts", +) + helm_remote("postgresql", repo_name="bitnami", repo_url="https://charts.bitnami.com/bitnami", From bb875cc45a7297a4b64b4cbe9472b71a079441d7 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 3 Feb 2022 15:40:29 -0500 Subject: [PATCH 02/17] Add boto --- poetry.lock | 24 ++++++++++++++++++------ pyproject.toml | 1 + 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index fc708b4..55646c0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -119,6 +119,14 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "boto" +version = "2.49.0" +description = "Amazon Web Services Library" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "bottle" version = "0.12.19" @@ -233,7 +241,7 @@ python-versions = "*" [[package]] name = "dramatiq" -version = "1.12.1" +version = "1.12.3" description = "Background Processing for Python 3." category = "main" optional = false @@ -244,8 +252,8 @@ prometheus-client = ">=0.2" redis = {version = ">=2.0,<5.0", optional = true, markers = "extra == \"redis\""} [package.extras] -all = ["redis (>=2.0,<5.0)", "pika (>=1.0,<2.0)", "watchdog", "pylibmc (>=1.5,<2.0)", "watchdog-gevent", "gevent (>=1.1)"] -dev = ["redis (>=2.0,<5.0)", "pika (>=1.0,<2.0)", "watchdog", "pylibmc (>=1.5,<2.0)", "watchdog-gevent", "gevent (>=1.1)", "alabaster", "sphinx (<1.8)", "sphinxcontrib-napoleon", "flake8", "flake8-bugbear", "flake8-quotes", "isort", "bumpversion", "hiredis", "twine", "wheel", "pytest", "pytest-benchmark", "pytest-cov", "tox"] +all = ["gevent (>=1.1)", "redis (>=2.0,<5.0)", "watchdog", "pika (>=1.0,<2.0)", "watchdog-gevent", "pylibmc (>=1.5,<2.0)"] +dev = ["gevent (>=1.1)", "redis (>=2.0,<5.0)", "watchdog", "pika (>=1.0,<2.0)", "watchdog-gevent", "pylibmc (>=1.5,<2.0)", "alabaster", "sphinx (<1.8)", "sphinxcontrib-napoleon", "flake8", "flake8-bugbear", "flake8-quotes", "isort", "bumpversion", "hiredis", "twine", "wheel", "pytest", "pytest-benchmark", "pytest-cov", "tox"] gevent = ["gevent (>=1.1)"] memcached = ["pylibmc (>=1.5,<2.0)"] rabbitmq = ["pika (>=1.0,<2.0)"] @@ -1127,7 +1135,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "955c3df01b275e9b4807190e468a2df4d3d18b6a45a7c1659599ef476b35be51" +content-hash = "ec661ab62e19249959e715522e29eb4bb17f036169c178838db8c9cfc9579820" [metadata.files] aiohttp = [ @@ -1239,6 +1247,10 @@ base58 = [ bitarray = [ {file = "bitarray-1.2.2.tar.gz", hash = "sha256:27a69ffcee3b868abab3ce8b17c69e02b63e722d4d64ffd91d659f81e9984954"}, ] +boto = [ + {file = "boto-2.49.0-py2.py3-none-any.whl", hash = "sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8"}, + {file = "boto-2.49.0.tar.gz", hash = "sha256:ea0d3b40a2d852767be77ca343b58a9e3a4b00d9db440efb8da74b4e58025e5a"}, +] bottle = [ {file = "bottle-0.12.19-py3-none-any.whl", hash = "sha256:f6b8a34fe9aa406f9813c02990db72ca69ce6a158b5b156d2c41f345016a723d"}, {file = "bottle-0.12.19.tar.gz", hash = "sha256:a9d73ffcbc6a1345ca2d7949638db46349f5b2b77dac65d6494d45c23628da2c"}, @@ -1352,8 +1364,8 @@ distlib = [ {file = "distlib-0.3.2.zip", hash = "sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736"}, ] dramatiq = [ - {file = "dramatiq-1.12.1-py3-none-any.whl", hash = "sha256:caf8f5baed6cb4afaf73b8379ffcd07f483de990b0f93f05d336d4efdcdfdecf"}, - {file = "dramatiq-1.12.1.tar.gz", hash = "sha256:0aabb8e9164a7b88b3799319bbe294f9823eaf8b9fa9f701dd45affc8ea57bbe"}, + {file = "dramatiq-1.12.3-py3-none-any.whl", hash = "sha256:eccb0f54d44ebd9e2c79e00d67b808397589a1a621ba7c5fd58df5fb6204a0a8"}, + {file = "dramatiq-1.12.3.tar.gz", hash = "sha256:380bd77b6b19d642f417b642935049ff71ddf4b4e57d821e4f55b92541430f21"}, ] eth-abi = [ {file = "eth_abi-2.1.1-py3-none-any.whl", hash = "sha256:78df5d2758247a8f0766a7cfcea4575bcfe568c34a33e6d05a72c328a9040444"}, diff --git a/pyproject.toml b/pyproject.toml index d616834..457288c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ psycopg2 = "^2.9.1" aiohttp = "^3.8.0" dramatiq = {extras = ["redis"], version = "^1.12.1"} pycoingecko = "^2.2.0" +boto = "^2.49.0" [tool.poetry.dev-dependencies] pre-commit = "^2.13.0" From 17dec2b203a5aee37b879880db654dfdfb053baf Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Fri, 4 Feb 2022 12:51:23 -0500 Subject: [PATCH 03/17] Expose localhost port --- Tiltfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tiltfile b/Tiltfile index 445399d..828ed9f 100644 --- a/Tiltfile +++ b/Tiltfile @@ -7,6 +7,12 @@ helm_remote("localstack", repo_url="https://localstack.github.io/helm-charts", ) +local_resource( + 'localstack-port-forward', + serve_cmd='kubectl port-forward --namespace default svc/localstack 4566:4566', + resource_deps=["localstack"] +) + helm_remote("postgresql", repo_name="bitnami", repo_url="https://charts.bitnami.com/bitnami", From e0fc9e7776278941b5c1535768d15a35106ba0f8 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Fri, 4 Feb 2022 13:09:46 -0500 Subject: [PATCH 04/17] Add a shell of a command to do the export --- cli.py | 8 ++++++++ mev | 23 +++++++++++++++-------- pyproject.toml | 1 + 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/cli.py b/cli.py index 3730e0f..2885de7 100644 --- a/cli.py +++ b/cli.py @@ -120,6 +120,14 @@ def fetch_all_prices(): write_prices(inspect_db_session, prices) +@cli.command() +@click.argument("after_block_number", type=int) +@click.argument("before_block_number", type=int) +def s3_export(after_block_number: int, before_block_number: int): + logger.info(after_block_number) + logger.info(before_block_number) + + @cli.command() @click.argument("after", type=click.DateTime(formats=["%Y-%m-%d", "%m-%d-%Y"])) @click.argument("before", type=click.DateTime(formats=["%Y-%m-%d", "%m-%d-%Y"])) diff --git a/mev b/mev index eb81fc6..65a3556 100755 --- a/mev +++ b/mev @@ -46,11 +46,11 @@ case "$1" in kubectl exec -ti deploy/mev-inspect -- ./listener $2 ;; backfill) - start_block_number=$2 - end_block_number=$3 + after_block_number=$2 + before_block_number=$3 - echo "Backfilling from $start_block_number to $end_block_number" - kubectl exec -ti deploy/mev-inspect -- poetry run enqueue-many-blocks $start_block_number $end_block_number + echo "Backfilling from $after_block_number to $before_block_number" + kubectl exec -ti deploy/mev-inspect -- poetry run enqueue-many-blocks $after_block_number $before_block_number ;; inspect) block_number=$2 @@ -58,11 +58,11 @@ case "$1" in kubectl exec -ti deploy/mev-inspect -- poetry run inspect-block $block_number ;; inspect-many) - start_block_number=$2 - end_block_number=$3 - echo "Inspecting from block $start_block_number to $end_block_number" + after_block_number=$2 + before_block_number=$3 + echo "Inspecting from block $after_block_number to $before_block_number" kubectl exec -ti deploy/mev-inspect -- \ - poetry run inspect-many-blocks $start_block_number $end_block_number + poetry run inspect-many-blocks $after_block_number $before_block_number ;; test) shift @@ -94,6 +94,13 @@ case "$1" in exit 1 esac ;; + s3-export) + after_block_number=$2 + before_block_number=$3 + + echo "Exporting from $after_block_number to $before_block_number" + kubectl exec -ti deploy/mev-inspect -- poetry run s3-export $after_block_number $before_block_number + ;; exec) shift kubectl exec -ti deploy/mev-inspect -- $@ diff --git a/pyproject.toml b/pyproject.toml index 457288c..4dd0b34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ enqueue-many-blocks = 'cli:enqueue_many_blocks_command' fetch-block = 'cli:fetch_block_command' fetch-all-prices = 'cli:fetch_all_prices' fetch-range = 'cli:fetch_range' +s3-export = 'cli:s3_export' [tool.black] exclude = ''' From 06c39d1495ce63a8215d843a8fc9521a961900f1 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Fri, 4 Feb 2022 14:36:30 -0500 Subject: [PATCH 05/17] Add boto3. Remove boto. Add a test connection to localstack --- cli.py | 3 ++ poetry.lock | 76 ++++++++++++++++++++++++++++++++++++++++++++------ pyproject.toml | 2 +- 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/cli.py b/cli.py index 2885de7..6021d31 100644 --- a/cli.py +++ b/cli.py @@ -13,6 +13,7 @@ from mev_inspect.inspector import MEVInspector from mev_inspect.prices import fetch_prices, fetch_prices_range from mev_inspect.queue.broker import connect_broker from mev_inspect.queue.tasks import inspect_many_blocks_task +from mev_inspect.s3_export import list_export_bucket RPC_URL_ENV = "RPC_URL" @@ -127,6 +128,8 @@ def s3_export(after_block_number: int, before_block_number: int): logger.info(after_block_number) logger.info(before_block_number) + print(list_export_bucket()) + @cli.command() @click.argument("after", type=click.DateTime(formats=["%Y-%m-%d", "%m-%d-%Y"])) diff --git a/poetry.lock b/poetry.lock index 55646c0..2299d5a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -120,12 +120,36 @@ optional = false python-versions = "*" [[package]] -name = "boto" -version = "2.49.0" -description = "Amazon Web Services Library" +name = "boto3" +version = "1.20.48" +description = "The AWS SDK for Python" category = "main" optional = false -python-versions = "*" +python-versions = ">= 3.6" + +[package.dependencies] +botocore = ">=1.23.48,<1.24.0" +jmespath = ">=0.7.1,<1.0.0" +s3transfer = ">=0.5.0,<0.6.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.23.48" +description = "Low-level, data-driven core of boto 3." +category = "main" +optional = false +python-versions = ">= 3.6" + +[package.dependencies] +jmespath = ">=0.7.1,<1.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<1.27" + +[package.extras] +crt = ["awscrt (==0.12.5)"] [[package]] name = "bottle" @@ -512,6 +536,14 @@ requirements_deprecated_finder = ["pipreqs", "pip-api"] colors = ["colorama (>=0.4.3,<0.5.0)"] plugins = ["setuptools"] +[[package]] +name = "jmespath" +version = "0.10.0" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + [[package]] name = "jsonschema" version = "3.2.0" @@ -874,7 +906,7 @@ termcolor = ">=1.1.0" name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "dev" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" @@ -963,6 +995,20 @@ lint = ["flake8 (==3.4.1)"] rust-backend = ["rusty-rlp (>=0.1.15,<0.2)"] test = ["hypothesis (==5.19.0)", "pytest (==5.4.3)", "tox (>=2.9.1,<3)"] +[[package]] +name = "s3transfer" +version = "0.5.1" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.6" + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + [[package]] name = "six" version = "1.16.0" @@ -1135,7 +1181,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "ec661ab62e19249959e715522e29eb4bb17f036169c178838db8c9cfc9579820" +content-hash = "063e246b07155c7bbc227ffd8a0d237d402a3eb00a804dbb389b67b7a0e35354" [metadata.files] aiohttp = [ @@ -1247,9 +1293,13 @@ base58 = [ bitarray = [ {file = "bitarray-1.2.2.tar.gz", hash = "sha256:27a69ffcee3b868abab3ce8b17c69e02b63e722d4d64ffd91d659f81e9984954"}, ] -boto = [ - {file = "boto-2.49.0-py2.py3-none-any.whl", hash = "sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8"}, - {file = "boto-2.49.0.tar.gz", hash = "sha256:ea0d3b40a2d852767be77ca343b58a9e3a4b00d9db440efb8da74b4e58025e5a"}, +boto3 = [ + {file = "boto3-1.20.48-py3-none-any.whl", hash = "sha256:1c6301d9676cb18f2b0feddec393e52b9d5fa8147e6fe9a1665e39fd9739efc3"}, + {file = "boto3-1.20.48.tar.gz", hash = "sha256:6a8111492a571aeefbac2e4b6df5ce38bdbc16c7d8326f2a60a61c86032c49b0"}, +] +botocore = [ + {file = "botocore-1.23.48-py3-none-any.whl", hash = "sha256:768acb9a2247155b974a4184b29be321242ef8f61827f4bb958e60f00e476e90"}, + {file = "botocore-1.23.48.tar.gz", hash = "sha256:8652c11ff05d11d6cea7096aca8df7f8eb87980469860036ff47e196e4625c96"}, ] bottle = [ {file = "bottle-0.12.19-py3-none-any.whl", hash = "sha256:f6b8a34fe9aa406f9813c02990db72ca69ce6a158b5b156d2c41f345016a723d"}, @@ -1556,6 +1606,10 @@ isort = [ {file = "isort-5.9.3-py3-none-any.whl", hash = "sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2"}, {file = "isort-5.9.3.tar.gz", hash = "sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899"}, ] +jmespath = [ + {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, + {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, +] jsonschema = [ {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, @@ -2021,6 +2075,10 @@ rlp = [ {file = "rlp-2.0.1-py2.py3-none-any.whl", hash = "sha256:52a57c9f53f03c88b189283734b397314288250cc4a3c4113e9e36e2ac6bdd16"}, {file = "rlp-2.0.1.tar.gz", hash = "sha256:665e8312750b3fc5f7002e656d05b9dcb6e93b6063df40d95c49ad90c19d1f0e"}, ] +s3transfer = [ + {file = "s3transfer-0.5.1-py3-none-any.whl", hash = "sha256:25c140f5c66aa79e1ac60be50dcd45ddc59e83895f062a3aab263b870102911f"}, + {file = "s3transfer-0.5.1.tar.gz", hash = "sha256:69d264d3e760e569b78aaa0f22c97e955891cd22e32b10c51f784eeda4d9d10a"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, diff --git a/pyproject.toml b/pyproject.toml index 4dd0b34..9b42e0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ psycopg2 = "^2.9.1" aiohttp = "^3.8.0" dramatiq = {extras = ["redis"], version = "^1.12.1"} pycoingecko = "^2.2.0" -boto = "^2.49.0" +boto3 = "^1.20.48" [tool.poetry.dev-dependencies] pre-commit = "^2.13.0" From d9439dfe27c0ea4564602cc2260c6f6373c8f766 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Fri, 4 Feb 2022 15:20:59 -0500 Subject: [PATCH 06/17] Run query. Export to S3 --- cli.py | 6 +-- mev_inspect/db.py | 2 +- mev_inspect/s3_export.py | 64 ++++++++++++++++++++++++ mev_inspect/{string_io.py => text_io.py} | 36 +++++++++++++ 4 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 mev_inspect/s3_export.py rename mev_inspect/{string_io.py => text_io.py} (52%) diff --git a/cli.py b/cli.py index 6021d31..8a78314 100644 --- a/cli.py +++ b/cli.py @@ -13,7 +13,7 @@ from mev_inspect.inspector import MEVInspector from mev_inspect.prices import fetch_prices, fetch_prices_range from mev_inspect.queue.broker import connect_broker from mev_inspect.queue.tasks import inspect_many_blocks_task -from mev_inspect.s3_export import list_export_bucket +from mev_inspect.s3_export import export_block_range, list_export_bucket RPC_URL_ENV = "RPC_URL" @@ -125,10 +125,10 @@ def fetch_all_prices(): @click.argument("after_block_number", type=int) @click.argument("before_block_number", type=int) def s3_export(after_block_number: int, before_block_number: int): - logger.info(after_block_number) - logger.info(before_block_number) + inspect_db_session = get_inspect_session() print(list_export_bucket()) + export_block_range(inspect_db_session, after_block_number, before_block_number) @cli.command() diff --git a/mev_inspect/db.py b/mev_inspect/db.py index dd7c66a..7fb1a97 100644 --- a/mev_inspect/db.py +++ b/mev_inspect/db.py @@ -4,7 +4,7 @@ from typing import Any, Iterable, List, Optional from sqlalchemy import create_engine, orm from sqlalchemy.orm import sessionmaker -from mev_inspect.string_io import StringIteratorIO +from mev_inspect.text_io import StringIteratorIO def get_trace_database_uri() -> Optional[str]: diff --git a/mev_inspect/s3_export.py b/mev_inspect/s3_export.py new file mode 100644 index 0000000..b27c472 --- /dev/null +++ b/mev_inspect/s3_export.py @@ -0,0 +1,64 @@ +import json + +import boto3 + +from mev_inspect.text_io import BytesIteratorIO + +MEV_SUMMARY_EXPORT_QUERY = """ + SELECT to_json(mev_summary) + FROM mev_summary + WHERE + block_number >= :after_block_number AND + block_number < :before_block_number + """ + + +def export_block_range( + inspect_db_session, after_block_number: int, before_block_number +) -> None: + client = get_s3_client() + bucket_name = get_export_bucket_name() + + mev_summary_json_results = inspect_db_session.execute( + statement=MEV_SUMMARY_EXPORT_QUERY, + params={ + "after_block_number": after_block_number, + "before_block_number": before_block_number, + }, + ) + + mev_summary_json_fileobj = BytesIteratorIO( + (f"{json.dumps(row)}\n".encode("utf-8") for (row,) in mev_summary_json_results) + ) + + key = f"mev_summary/{after_block_number}-{before_block_number}.json" + + client.upload_fileobj( + mev_summary_json_fileobj, + Bucket=bucket_name, + Key=key, + ) + + +def list_export_bucket(): + client = get_s3_client() + return client.list_objects_v2( + Bucket=get_export_bucket_name(), + Prefix="/", + ) + + +# TODO - replaced by ConfigMap +def get_export_bucket_name() -> str: + return "local-export" + + +# TODO +def get_s3_client(): + return boto3.client( + "s3", + region_name="us-east-1", + endpoint_url="http://localstack:4566", + aws_access_key_id="test", + aws_secret_access_key="test", + ) diff --git a/mev_inspect/string_io.py b/mev_inspect/text_io.py similarity index 52% rename from mev_inspect/string_io.py rename to mev_inspect/text_io.py index 37efb5f..e787e72 100644 --- a/mev_inspect/string_io.py +++ b/mev_inspect/text_io.py @@ -38,3 +38,39 @@ class StringIteratorIO(io.TextIOBase): n -= len(m) line.append(m) return "".join(line) + + +class BytesIteratorIO(io.BufferedIOBase): + def __init__(self, iter: Iterator[bytes]): + self._iter = iter + self._buff = b"" + + def readable(self) -> bool: + return True + + def _read1(self, n: Optional[int] = None) -> bytes: + while not self._buff: + try: + self._buff = next(self._iter) + except StopIteration: + break + ret = self._buff[:n] + self._buff = self._buff[len(ret) :] + return ret + + def read(self, n: Optional[int] = None) -> bytes: + line = [] + if n is None or n < 0: + while True: + m = self._read1() + if not m: + break + line.append(m) + else: + while n > 0: + m = self._read1(n) + if not m: + break + n -= len(m) + line.append(m) + return b"".join(line) From b1d4cb852b9db93113c96494df8803bc4fbaff63 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Fri, 4 Feb 2022 15:30:20 -0500 Subject: [PATCH 07/17] Add some logging. Remove unused list function --- cli.py | 4 +--- mev_inspect/s3_export.py | 13 +++++-------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/cli.py b/cli.py index 8a78314..b25d2a6 100644 --- a/cli.py +++ b/cli.py @@ -13,7 +13,7 @@ from mev_inspect.inspector import MEVInspector from mev_inspect.prices import fetch_prices, fetch_prices_range from mev_inspect.queue.broker import connect_broker from mev_inspect.queue.tasks import inspect_many_blocks_task -from mev_inspect.s3_export import export_block_range, list_export_bucket +from mev_inspect.s3_export import export_block_range RPC_URL_ENV = "RPC_URL" @@ -126,8 +126,6 @@ def fetch_all_prices(): @click.argument("before_block_number", type=int) def s3_export(after_block_number: int, before_block_number: int): inspect_db_session = get_inspect_session() - - print(list_export_bucket()) export_block_range(inspect_db_session, after_block_number, before_block_number) diff --git a/mev_inspect/s3_export.py b/mev_inspect/s3_export.py index b27c472..8b7aba1 100644 --- a/mev_inspect/s3_export.py +++ b/mev_inspect/s3_export.py @@ -1,4 +1,5 @@ import json +import logging import boto3 @@ -12,6 +13,8 @@ MEV_SUMMARY_EXPORT_QUERY = """ block_number < :before_block_number """ +logger = logging.getLogger(__name__) + def export_block_range( inspect_db_session, after_block_number: int, before_block_number @@ -39,13 +42,7 @@ def export_block_range( Key=key, ) - -def list_export_bucket(): - client = get_s3_client() - return client.list_objects_v2( - Bucket=get_export_bucket_name(), - Prefix="/", - ) + logger.info(f"Exported to {key}") # TODO - replaced by ConfigMap @@ -53,7 +50,7 @@ def get_export_bucket_name() -> str: return "local-export" -# TODO +# TODO - handle for production def get_s3_client(): return boto3.client( "s3", From 00dba743d9013e5e8b3029e2148fdb38c371f042 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Fri, 4 Feb 2022 16:21:58 -0500 Subject: [PATCH 08/17] ConfigMap --- Tiltfile | 4 ++++ k8s/mev-inspect/templates/deployment.yaml | 5 +++++ mev_inspect/s3_export.py | 10 +++------- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Tiltfile b/Tiltfile index 828ed9f..3b3c8a9 100644 --- a/Tiltfile +++ b/Tiltfile @@ -78,3 +78,7 @@ local_resource( serve_cmd='kubectl port-forward --namespace default svc/postgresql 5432:5432', resource_deps=["postgresql"] ) + +k8s_yaml(configmap_from_dict("mev-inspect-export", inputs = { + "export-bucket-name" : "local-export", +})) diff --git a/k8s/mev-inspect/templates/deployment.yaml b/k8s/mev-inspect/templates/deployment.yaml index fa2123f..329bb84 100644 --- a/k8s/mev-inspect/templates/deployment.yaml +++ b/k8s/mev-inspect/templates/deployment.yaml @@ -91,6 +91,11 @@ spec: name: mev-inspect-listener-healthcheck key: url optional: true + - name: EXPORT_BUCKET_NAME + valueFrom: + configMapKeyRef: + name: mev-inspect-export + key: export-bucket-name {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/mev_inspect/s3_export.py b/mev_inspect/s3_export.py index 8b7aba1..2710af8 100644 --- a/mev_inspect/s3_export.py +++ b/mev_inspect/s3_export.py @@ -1,5 +1,6 @@ import json import logging +import os import boto3 @@ -7,7 +8,7 @@ from mev_inspect.text_io import BytesIteratorIO MEV_SUMMARY_EXPORT_QUERY = """ SELECT to_json(mev_summary) - FROM mev_summary + FROM mev_summary WHERE block_number >= :after_block_number AND block_number < :before_block_number @@ -20,7 +21,7 @@ def export_block_range( inspect_db_session, after_block_number: int, before_block_number ) -> None: client = get_s3_client() - bucket_name = get_export_bucket_name() + bucket_name = os.environ["EXPORT_BUCKET_NAME"] mev_summary_json_results = inspect_db_session.execute( statement=MEV_SUMMARY_EXPORT_QUERY, @@ -45,11 +46,6 @@ def export_block_range( logger.info(f"Exported to {key}") -# TODO - replaced by ConfigMap -def get_export_bucket_name() -> str: - return "local-export" - - # TODO - handle for production def get_s3_client(): return boto3.client( From 040be01e9d9600f8ab4b61faeee67d3d950be9e5 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Fri, 4 Feb 2022 16:30:34 -0500 Subject: [PATCH 09/17] Set aws creds through environment variables locally --- Tiltfile | 27 +++++++++++++++++-- .../templates/deployment.yaml | 4 +++ k8s/mev-inspect/templates/deployment.yaml | 4 +++ mev_inspect/s3_export.py | 20 +++++++------- 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/Tiltfile b/Tiltfile index 3b3c8a9..829febf 100644 --- a/Tiltfile +++ b/Tiltfile @@ -53,7 +53,20 @@ docker_build("mev-inspect-py", ".", trigger="./pyproject.toml"), ], ) -k8s_yaml(helm('./k8s/mev-inspect', name='mev-inspect')) +k8s_yaml(helm( + './k8s/mev-inspect', + name='mev-inspect', + set=[ + "extraEnv[0].name=AWS_ACCESS_KEY_ID", + "extraEnv[0].value=foobar", + "extraEnv[1].name=AWS_SECRET_ACCESS_KEY", + "extraEnv[1].value=foobar", + "extraEnv[2].name=AWS_REGION", + "extraEnv[2].value=us-east-1", + "extraEnv[3].name=AWS_ENDPOINT_URL", + "extraEnv[3].value=http://localstack:4566", + ], +)) k8s_resource( workload="mev-inspect", resource_deps=["postgresql", "redis-master"], @@ -62,7 +75,17 @@ k8s_resource( k8s_yaml(helm( './k8s/mev-inspect-workers', name='mev-inspect-workers', - set=["replicaCount=1"], + set=[ + "extraEnv[0].name=AWS_ACCESS_KEY_ID", + "extraEnv[0].value=foobar", + "extraEnv[1].name=AWS_SECRET_ACCESS_KEY", + "extraEnv[1].value=foobar", + "extraEnv[2].name=AWS_REGION", + "extraEnv[2].value=us-east-1", + "extraEnv[3].name=AWS_ENDPOINT_URL", + "extraEnv[3].value=http://localstack:4566", + "replicaCount=1", + ], )) k8s_resource( workload="mev-inspect-workers", diff --git a/k8s/mev-inspect-workers/templates/deployment.yaml b/k8s/mev-inspect-workers/templates/deployment.yaml index 8b979aa..19486d1 100644 --- a/k8s/mev-inspect-workers/templates/deployment.yaml +++ b/k8s/mev-inspect-workers/templates/deployment.yaml @@ -91,6 +91,10 @@ spec: name: mev-inspect-listener-healthcheck key: url optional: true + {{- range .Values.extraEnv }} + - name: {{ .name }} + value: {{ .value }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/k8s/mev-inspect/templates/deployment.yaml b/k8s/mev-inspect/templates/deployment.yaml index 329bb84..fba11e0 100644 --- a/k8s/mev-inspect/templates/deployment.yaml +++ b/k8s/mev-inspect/templates/deployment.yaml @@ -96,6 +96,10 @@ spec: configMapKeyRef: name: mev-inspect-export key: export-bucket-name + {{- range .Values.extraEnv }} + - name: {{ .name }} + value: {{ .value }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/mev_inspect/s3_export.py b/mev_inspect/s3_export.py index 2710af8..272d6ad 100644 --- a/mev_inspect/s3_export.py +++ b/mev_inspect/s3_export.py @@ -1,11 +1,13 @@ import json import logging import os +from typing import Optional import boto3 from mev_inspect.text_io import BytesIteratorIO +AWS_ENDPOINT_URL_ENV = "AWS_ENDPOINT_URL" MEV_SUMMARY_EXPORT_QUERY = """ SELECT to_json(mev_summary) FROM mev_summary @@ -20,8 +22,8 @@ logger = logging.getLogger(__name__) def export_block_range( inspect_db_session, after_block_number: int, before_block_number ) -> None: + export_bucket_name = os.environ["EXPORT_BUCKET_NAME"] client = get_s3_client() - bucket_name = os.environ["EXPORT_BUCKET_NAME"] mev_summary_json_results = inspect_db_session.execute( statement=MEV_SUMMARY_EXPORT_QUERY, @@ -39,19 +41,17 @@ def export_block_range( client.upload_fileobj( mev_summary_json_fileobj, - Bucket=bucket_name, + Bucket=export_bucket_name, Key=key, ) logger.info(f"Exported to {key}") -# TODO - handle for production def get_s3_client(): - return boto3.client( - "s3", - region_name="us-east-1", - endpoint_url="http://localstack:4566", - aws_access_key_id="test", - aws_secret_access_key="test", - ) + endpoint_url = get_endpoint_url() + return boto3.client("s3", endpoint_url=endpoint_url) + + +def get_endpoint_url() -> Optional[str]: + return os.environ.get(AWS_ENDPOINT_URL_ENV) From 462bff387a18fcd5880940418e924069d424568a Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Fri, 4 Feb 2022 16:32:42 -0500 Subject: [PATCH 10/17] Break env piece into a function --- mev_inspect/s3_export.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mev_inspect/s3_export.py b/mev_inspect/s3_export.py index 272d6ad..1e979a1 100644 --- a/mev_inspect/s3_export.py +++ b/mev_inspect/s3_export.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) def export_block_range( inspect_db_session, after_block_number: int, before_block_number ) -> None: - export_bucket_name = os.environ["EXPORT_BUCKET_NAME"] + export_bucket_name = get_export_bucket_name() client = get_s3_client() mev_summary_json_results = inspect_db_session.execute( @@ -55,3 +55,7 @@ def get_s3_client(): def get_endpoint_url() -> Optional[str]: return os.environ.get(AWS_ENDPOINT_URL_ENV) + + +def get_export_bucket_name() -> str: + return os.environ["EXPORT_BUCKET_NAME"] From dbebb57b9c14a3b2be0568f9fa54e14773550cc0 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Tue, 8 Feb 2022 13:15:01 -0500 Subject: [PATCH 11/17] Tiltfile comments and services constraint --- Tiltfile | 75 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/Tiltfile b/Tiltfile index 829febf..b0bfd71 100644 --- a/Tiltfile +++ b/Tiltfile @@ -2,17 +2,6 @@ load("ext://helm_remote", "helm_remote") load("ext://secret", "secret_from_dict") load("ext://configmap", "configmap_from_dict") -helm_remote("localstack", - repo_name="localstack-charts", - repo_url="https://localstack.github.io/helm-charts", -) - -local_resource( - 'localstack-port-forward', - serve_cmd='kubectl port-forward --namespace default svc/localstack 4566:4566', - resource_deps=["localstack"] -) - helm_remote("postgresql", repo_name="bitnami", repo_url="https://charts.bitnami.com/bitnami", @@ -72,21 +61,6 @@ k8s_resource( resource_deps=["postgresql", "redis-master"], ) -k8s_yaml(helm( - './k8s/mev-inspect-workers', - name='mev-inspect-workers', - set=[ - "extraEnv[0].name=AWS_ACCESS_KEY_ID", - "extraEnv[0].value=foobar", - "extraEnv[1].name=AWS_SECRET_ACCESS_KEY", - "extraEnv[1].value=foobar", - "extraEnv[2].name=AWS_REGION", - "extraEnv[2].value=us-east-1", - "extraEnv[3].name=AWS_ENDPOINT_URL", - "extraEnv[3].value=http://localstack:4566", - "replicaCount=1", - ], -)) k8s_resource( workload="mev-inspect-workers", resource_deps=["postgresql", "redis-master"], @@ -102,6 +76,49 @@ local_resource( resource_deps=["postgresql"] ) -k8s_yaml(configmap_from_dict("mev-inspect-export", inputs = { - "export-bucket-name" : "local-export", -})) +# Uncomment to use local S3 exports +# +#k8s_yaml(configmap_from_dict("mev-inspect-export", inputs = { +# "export-bucket-name" : "local-export", +#})) +# +#helm_remote("localstack", +# repo_name="localstack-charts", +# repo_url="https://localstack.github.io/helm-charts", +#) +# +#local_resource( +# 'localstack-port-forward', +# serve_cmd='kubectl port-forward --namespace default svc/localstack 4566:4566', +# resource_deps=["localstack"] +#) +# +#k8s_yaml(helm( +# './k8s/mev-inspect-workers', +# name='mev-inspect-workers', +# set=[ +# "extraEnv[0].name=AWS_ACCESS_KEY_ID", +# "extraEnv[0].value=foobar", +# "extraEnv[1].name=AWS_SECRET_ACCESS_KEY", +# "extraEnv[1].value=foobar", +# "extraEnv[2].name=AWS_REGION", +# "extraEnv[2].value=us-east-1", +# "extraEnv[3].name=AWS_ENDPOINT_URL", +# "extraEnv[3].value=http://localstack:4566", +# "replicaCount=1", +# ], +#)) +#k8s_yaml(helm( +# './k8s/mev-inspect', +# name='mev-inspect', +# set=[ +# "extraEnv[0].name=AWS_ACCESS_KEY_ID", +# "extraEnv[0].value=foobar", +# "extraEnv[1].name=AWS_SECRET_ACCESS_KEY", +# "extraEnv[1].value=foobar", +# "extraEnv[2].name=AWS_REGION", +# "extraEnv[2].value=us-east-1", +# "extraEnv[3].name=AWS_ENDPOINT_URL", +# "extraEnv[3].value=http://localstack:4566", +# ], +#)) From 751059c53444fe78f78a46c8bb7f3ac35d72e091 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Tue, 8 Feb 2022 13:18:28 -0500 Subject: [PATCH 12/17] Remove some comments --- Tiltfile | 60 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/Tiltfile b/Tiltfile index b0bfd71..17f916d 100644 --- a/Tiltfile +++ b/Tiltfile @@ -92,33 +92,33 @@ local_resource( # serve_cmd='kubectl port-forward --namespace default svc/localstack 4566:4566', # resource_deps=["localstack"] #) -# -#k8s_yaml(helm( -# './k8s/mev-inspect-workers', -# name='mev-inspect-workers', -# set=[ -# "extraEnv[0].name=AWS_ACCESS_KEY_ID", -# "extraEnv[0].value=foobar", -# "extraEnv[1].name=AWS_SECRET_ACCESS_KEY", -# "extraEnv[1].value=foobar", -# "extraEnv[2].name=AWS_REGION", -# "extraEnv[2].value=us-east-1", -# "extraEnv[3].name=AWS_ENDPOINT_URL", -# "extraEnv[3].value=http://localstack:4566", -# "replicaCount=1", -# ], -#)) -#k8s_yaml(helm( -# './k8s/mev-inspect', -# name='mev-inspect', -# set=[ -# "extraEnv[0].name=AWS_ACCESS_KEY_ID", -# "extraEnv[0].value=foobar", -# "extraEnv[1].name=AWS_SECRET_ACCESS_KEY", -# "extraEnv[1].value=foobar", -# "extraEnv[2].name=AWS_REGION", -# "extraEnv[2].value=us-east-1", -# "extraEnv[3].name=AWS_ENDPOINT_URL", -# "extraEnv[3].value=http://localstack:4566", -# ], -#)) + +k8s_yaml(helm( + './k8s/mev-inspect-workers', + name='mev-inspect-workers', + set=[ + "extraEnv[0].name=AWS_ACCESS_KEY_ID", + "extraEnv[0].value=foobar", + "extraEnv[1].name=AWS_SECRET_ACCESS_KEY", + "extraEnv[1].value=foobar", + "extraEnv[2].name=AWS_REGION", + "extraEnv[2].value=us-east-1", + "extraEnv[3].name=AWS_ENDPOINT_URL", + "extraEnv[3].value=http://localstack:4566", + "replicaCount=1", + ], +)) +k8s_yaml(helm( + './k8s/mev-inspect', + name='mev-inspect', + set=[ + "extraEnv[0].name=AWS_ACCESS_KEY_ID", + "extraEnv[0].value=foobar", + "extraEnv[1].name=AWS_SECRET_ACCESS_KEY", + "extraEnv[1].value=foobar", + "extraEnv[2].name=AWS_REGION", + "extraEnv[2].value=us-east-1", + "extraEnv[3].name=AWS_ENDPOINT_URL", + "extraEnv[3].value=http://localstack:4566", + ], +) From aa5c90ae96d747a827a91749179bc3349a532192 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Tue, 8 Feb 2022 15:00:19 -0500 Subject: [PATCH 13/17] only one mev inpect helml --- Tiltfile | 79 +++++++++++++++++++++++--------------------------------- 1 file changed, 33 insertions(+), 46 deletions(-) diff --git a/Tiltfile b/Tiltfile index 17f916d..4147dfa 100644 --- a/Tiltfile +++ b/Tiltfile @@ -42,6 +42,7 @@ docker_build("mev-inspect-py", ".", trigger="./pyproject.toml"), ], ) + k8s_yaml(helm( './k8s/mev-inspect', name='mev-inspect', @@ -56,6 +57,23 @@ k8s_yaml(helm( "extraEnv[3].value=http://localstack:4566", ], )) + +k8s_yaml(helm( + './k8s/mev-inspect-workers', + name='mev-inspect-workers', + set=[ + "extraEnv[0].name=AWS_ACCESS_KEY_ID", + "extraEnv[0].value=foobar", + "extraEnv[1].name=AWS_SECRET_ACCESS_KEY", + "extraEnv[1].value=foobar", + "extraEnv[2].name=AWS_REGION", + "extraEnv[2].value=us-east-1", + "extraEnv[3].name=AWS_ENDPOINT_URL", + "extraEnv[3].value=http://localstack:4566", + "replicaCount=1", + ], +)) + k8s_resource( workload="mev-inspect", resource_deps=["postgresql", "redis-master"], @@ -76,49 +94,18 @@ local_resource( resource_deps=["postgresql"] ) -# Uncomment to use local S3 exports -# -#k8s_yaml(configmap_from_dict("mev-inspect-export", inputs = { -# "export-bucket-name" : "local-export", -#})) -# -#helm_remote("localstack", -# repo_name="localstack-charts", -# repo_url="https://localstack.github.io/helm-charts", -#) -# -#local_resource( -# 'localstack-port-forward', -# serve_cmd='kubectl port-forward --namespace default svc/localstack 4566:4566', -# resource_deps=["localstack"] -#) - -k8s_yaml(helm( - './k8s/mev-inspect-workers', - name='mev-inspect-workers', - set=[ - "extraEnv[0].name=AWS_ACCESS_KEY_ID", - "extraEnv[0].value=foobar", - "extraEnv[1].name=AWS_SECRET_ACCESS_KEY", - "extraEnv[1].value=foobar", - "extraEnv[2].name=AWS_REGION", - "extraEnv[2].value=us-east-1", - "extraEnv[3].name=AWS_ENDPOINT_URL", - "extraEnv[3].value=http://localstack:4566", - "replicaCount=1", - ], -)) -k8s_yaml(helm( - './k8s/mev-inspect', - name='mev-inspect', - set=[ - "extraEnv[0].name=AWS_ACCESS_KEY_ID", - "extraEnv[0].value=foobar", - "extraEnv[1].name=AWS_SECRET_ACCESS_KEY", - "extraEnv[1].value=foobar", - "extraEnv[2].name=AWS_REGION", - "extraEnv[2].value=us-east-1", - "extraEnv[3].name=AWS_ENDPOINT_URL", - "extraEnv[3].value=http://localstack:4566", - ], -) +# if using local S3 exports +# k8s_yaml(configmap_from_dict("mev-inspect-export", inputs = { +# "export-bucket-name" : "local-export", +# })) +# +# helm_remote("localstack", +# repo_name="localstack-charts", +# repo_url="https://localstack.github.io/helm-charts", +# ) +# +# local_resource( +# 'localstack-port-forward', +# serve_cmd='kubectl port-forward --namespace default svc/localstack 4566:4566', +# resource_deps=["localstack"] +# ) From 001b6e2b850837d8990d29bfbc3a9673854e329f Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Tue, 8 Feb 2022 15:00:31 -0500 Subject: [PATCH 14/17] Add a flashbots prefix --- mev_inspect/s3_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mev_inspect/s3_export.py b/mev_inspect/s3_export.py index 1e979a1..c7ee60b 100644 --- a/mev_inspect/s3_export.py +++ b/mev_inspect/s3_export.py @@ -37,7 +37,7 @@ def export_block_range( (f"{json.dumps(row)}\n".encode("utf-8") for (row,) in mev_summary_json_results) ) - key = f"mev_summary/{after_block_number}-{before_block_number}.json" + key = f"mev_summary/flashbots_{after_block_number}_{before_block_number}.json" client.upload_fileobj( mev_summary_json_fileobj, From c079ac9aa6563a8ea293eb24d35144d250da97c5 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Tue, 8 Feb 2022 15:36:45 -0500 Subject: [PATCH 15/17] Add region for the export bucket --- Tiltfile | 8 +++++--- k8s/mev-inspect-workers/templates/deployment.yaml | 10 ++++++++++ k8s/mev-inspect/templates/deployment.yaml | 5 +++++ mev_inspect/s3_export.py | 10 +++++++++- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/Tiltfile b/Tiltfile index 4147dfa..7b324c7 100644 --- a/Tiltfile +++ b/Tiltfile @@ -97,11 +97,13 @@ local_resource( # if using local S3 exports # k8s_yaml(configmap_from_dict("mev-inspect-export", inputs = { # "export-bucket-name" : "local-export", +# "export-bucket-region": "us-east-1", # })) # -# helm_remote("localstack", -# repo_name="localstack-charts", -# repo_url="https://localstack.github.io/helm-charts", +# helm_remote( +# "localstack", +# repo_name="localstack-charts", +# repo_url="https://localstack.github.io/helm-charts", # ) # # local_resource( diff --git a/k8s/mev-inspect-workers/templates/deployment.yaml b/k8s/mev-inspect-workers/templates/deployment.yaml index 19486d1..63808da 100644 --- a/k8s/mev-inspect-workers/templates/deployment.yaml +++ b/k8s/mev-inspect-workers/templates/deployment.yaml @@ -91,6 +91,16 @@ spec: name: mev-inspect-listener-healthcheck key: url optional: true + - name: EXPORT_BUCKET_NAME + valueFrom: + configMapKeyRef: + name: mev-inspect-export + key: export-bucket-name + - name: EXPORT_BUCKET_REGION + valueFrom: + configMapKeyRef: + name: mev-inspect-export + key: export-bucket-region {{- range .Values.extraEnv }} - name: {{ .name }} value: {{ .value }} diff --git a/k8s/mev-inspect/templates/deployment.yaml b/k8s/mev-inspect/templates/deployment.yaml index fba11e0..4dda409 100644 --- a/k8s/mev-inspect/templates/deployment.yaml +++ b/k8s/mev-inspect/templates/deployment.yaml @@ -96,6 +96,11 @@ spec: configMapKeyRef: name: mev-inspect-export key: export-bucket-name + - name: EXPORT_BUCKET_REGION + valueFrom: + configMapKeyRef: + name: mev-inspect-export + key: export-bucket-region {{- range .Values.extraEnv }} - name: {{ .name }} value: {{ .value }} diff --git a/mev_inspect/s3_export.py b/mev_inspect/s3_export.py index c7ee60b..8477287 100644 --- a/mev_inspect/s3_export.py +++ b/mev_inspect/s3_export.py @@ -50,7 +50,11 @@ def export_block_range( def get_s3_client(): endpoint_url = get_endpoint_url() - return boto3.client("s3", endpoint_url=endpoint_url) + return boto3.client( + "s3", + endpoint_url=endpoint_url, + region_name=get_export_bucket_region(), + ) def get_endpoint_url() -> Optional[str]: @@ -59,3 +63,7 @@ def get_endpoint_url() -> Optional[str]: def get_export_bucket_name() -> str: return os.environ["EXPORT_BUCKET_NAME"] + + +def get_export_bucket_region() -> str: + return os.environ["EXPORT_BUCKET_REGION"] From 4dbe6ed2d7d85b3331a5fcd01b6f3891021fd599 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Wed, 9 Feb 2022 08:42:57 -0500 Subject: [PATCH 16/17] Pass through AWS creds as well. Turn into a secret. Make all optional for folks not using the export --- Tiltfile | 4 +++- .../templates/deployment.yaml | 18 ++++++++++++++++-- k8s/mev-inspect/templates/deployment.yaml | 18 ++++++++++++++++-- mev_inspect/s3_export.py | 10 ++++++++++ 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/Tiltfile b/Tiltfile index 7b324c7..0ccc758 100644 --- a/Tiltfile +++ b/Tiltfile @@ -95,9 +95,11 @@ local_resource( ) # if using local S3 exports -# k8s_yaml(configmap_from_dict("mev-inspect-export", inputs = { +# k8s_yaml(secret_from_dict("mev-inspect-export", inputs = { # "export-bucket-name" : "local-export", # "export-bucket-region": "us-east-1", +# "export-aws-access-key-id": "foobar", +# "export-aws-secret-access-key": "foobar", # })) # # helm_remote( diff --git a/k8s/mev-inspect-workers/templates/deployment.yaml b/k8s/mev-inspect-workers/templates/deployment.yaml index 63808da..0a3238f 100644 --- a/k8s/mev-inspect-workers/templates/deployment.yaml +++ b/k8s/mev-inspect-workers/templates/deployment.yaml @@ -93,14 +93,28 @@ spec: optional: true - name: EXPORT_BUCKET_NAME valueFrom: - configMapKeyRef: + secretKeyRef: name: mev-inspect-export key: export-bucket-name + optional: true - name: EXPORT_BUCKET_REGION valueFrom: - configMapKeyRef: + secretKeyRef: name: mev-inspect-export key: export-bucket-region + optional: true + - name: EXPORT_AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: mev-inspect-export + key: export-aws-access-key-id + optional: true + - name: EXPORT_AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mev-inspect-export + key: export-aws-secret-access-key + optional: true {{- range .Values.extraEnv }} - name: {{ .name }} value: {{ .value }} diff --git a/k8s/mev-inspect/templates/deployment.yaml b/k8s/mev-inspect/templates/deployment.yaml index 4dda409..b9da25e 100644 --- a/k8s/mev-inspect/templates/deployment.yaml +++ b/k8s/mev-inspect/templates/deployment.yaml @@ -93,14 +93,28 @@ spec: optional: true - name: EXPORT_BUCKET_NAME valueFrom: - configMapKeyRef: + secretKeyRef: name: mev-inspect-export key: export-bucket-name + optional: true - name: EXPORT_BUCKET_REGION valueFrom: - configMapKeyRef: + secretKeyRef: name: mev-inspect-export key: export-bucket-region + optional: true + - name: EXPORT_AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: mev-inspect-export + key: export-aws-access-key-id + optional: true + - name: EXPORT_AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: mev-inspect-export + key: export-aws-secret-access-key + optional: true {{- range .Values.extraEnv }} - name: {{ .name }} value: {{ .value }} diff --git a/mev_inspect/s3_export.py b/mev_inspect/s3_export.py index 8477287..e1db97f 100644 --- a/mev_inspect/s3_export.py +++ b/mev_inspect/s3_export.py @@ -54,6 +54,8 @@ def get_s3_client(): "s3", endpoint_url=endpoint_url, region_name=get_export_bucket_region(), + aws_access_key_id=get_export_aws_access_key_id(), + aws_secret_access_key=get_export_aws_secret_access_key(), ) @@ -67,3 +69,11 @@ def get_export_bucket_name() -> str: def get_export_bucket_region() -> str: return os.environ["EXPORT_BUCKET_REGION"] + + +def get_export_aws_access_key_id() -> str: + return os.environ["EXPORT_AWS_ACCESS_KEY_ID"] + + +def get_export_aws_secret_access_key() -> str: + return os.environ["EXPORT_AWS_SECRET_ACCESS_KEY"] From 88b5e0ce2a90ae66f0f49536d6699e51a3b2152d Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Fri, 11 Feb 2022 11:19:59 -0500 Subject: [PATCH 17/17] Move ENV names to variables. Make region and keys optional --- mev_inspect/s3_export.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/mev_inspect/s3_export.py b/mev_inspect/s3_export.py index e1db97f..30ea439 100644 --- a/mev_inspect/s3_export.py +++ b/mev_inspect/s3_export.py @@ -8,6 +8,11 @@ import boto3 from mev_inspect.text_io import BytesIteratorIO AWS_ENDPOINT_URL_ENV = "AWS_ENDPOINT_URL" +EXPORT_BUCKET_NAME_ENV = "EXPORT_BUCKET_NAME" +EXPORT_BUCKET_REGION_ENV = "EXPORT_BUCKET_REGION" +EXPORT_AWS_ACCESS_KEY_ID_ENV = "EXPORT_AWS_ACCESS_KEY_ID" +EXPORT_AWS_SECRET_ACCESS_KEY_ENV = "EXPORT_AWS_SECRET_ACCESS_KEY" + MEV_SUMMARY_EXPORT_QUERY = """ SELECT to_json(mev_summary) FROM mev_summary @@ -64,16 +69,16 @@ def get_endpoint_url() -> Optional[str]: def get_export_bucket_name() -> str: - return os.environ["EXPORT_BUCKET_NAME"] + return os.environ[EXPORT_BUCKET_NAME_ENV] -def get_export_bucket_region() -> str: - return os.environ["EXPORT_BUCKET_REGION"] +def get_export_bucket_region() -> Optional[str]: + return os.environ.get(EXPORT_BUCKET_REGION_ENV) -def get_export_aws_access_key_id() -> str: - return os.environ["EXPORT_AWS_ACCESS_KEY_ID"] +def get_export_aws_access_key_id() -> Optional[str]: + return os.environ.get(EXPORT_AWS_ACCESS_KEY_ID_ENV) -def get_export_aws_secret_access_key() -> str: - return os.environ["EXPORT_AWS_SECRET_ACCESS_KEY"] +def get_export_aws_secret_access_key() -> Optional[str]: + return os.environ.get(EXPORT_AWS_SECRET_ACCESS_KEY_ENV)