diff --git a/cli.py b/cli.py index 1f13778..c9bf8a1 100644 --- a/cli.py +++ b/cli.py @@ -134,14 +134,12 @@ def fetch_range(after: datetime, before: datetime): @cli.command() @click.argument("block_number", type=int) -@click.argument("bucket", type=str) -@click.argument("filepath", type=str) -@click.argument("region", type=str) -def s3_export_command(block_number: int, bucket: str, filepath: str, region: str): +@click.argument("uri", type=str) +def s3_export_command(block_number: int, uri: str): inspect_db_session = get_inspect_session() logger.info(f"Exporting block {block_number}") - s3_export(inspect_db_session, block_number, bucket, filepath, region) + s3_export(inspect_db_session, uri) return None @@ -149,17 +147,15 @@ def s3_export_command(block_number: int, bucket: str, filepath: str, region: str @cli.command() @click.argument("after_block", type=int) @click.argument("before_block", type=int) -@click.argument("bucket", type=str) -@click.argument("filepath_base", type=str) -@click.argument("region", type=str) +@click.argument("base_uri", type=str) def s3_export_many_command( - after_block: int, before_block: int, bucket: str, filepath_base: str, region: str + after_block: int, before_block: int, base_uri: str ): inspect_db_session = get_inspect_session() logger.info(f"Exporting blocks {after_block} to {before_block}") s3_export_many( - inspect_db_session, after_block, before_block, bucket, filepath_base, region + inspect_db_session, after_block, before_block, base_uri ) return None diff --git a/mev b/mev index 91d1a6b..efe341a 100755 --- a/mev +++ b/mev @@ -99,23 +99,19 @@ case "$1" in case "$1" in s3-export) block_number=$2 - bucket=$3 - filepath=$4 - region=$5 + uri=$3 echo "Exporting block" kubectl exec -ti deploy/mev-inspect -- \ - poetry run s3-export $block_number $bucket $filepath $region + poetry run s3-export $block_number $uri ;; s3-export-many) after_block=$2 before_block=$3 - bucket=$4 - filepath_base=$5 - region=$6 + base_uri=$4 echo "Exporting blocks" kubectl exec -ti deploy/mev-inspect -- \ - poetry run s3-export-many $after_block $before_block $bucket $filepath_base $region + poetry run s3-export-many $after_block $before_block $base_uri ;; *) echo "export usage: "$1" {s3-export}" diff --git a/mev_inspect/export.py b/mev_inspect/export.py index 5cf96d9..d8696d5 100644 --- a/mev_inspect/export.py +++ b/mev_inspect/export.py @@ -5,12 +5,10 @@ from mev_inspect.crud.latest_s3_block import ( def s3_export( - db_session, block_number: int, bucket: str, filepath: str, region: str + db_session, block_number: int, uri: str, ) -> None: """Export block to S3""" - uri = _get_uri(db_session, bucket, filepath, region) - latest_s3_block = find_latest_s3_block(db_session) if latest_s3_block is not None: @@ -32,9 +30,7 @@ def s3_export_many( db_session, after_block: int, before_block: int, - bucket: str, - filepath_base: str, - region: str, + base_uri: str, ) -> None: """Export block range to S3""" @@ -43,8 +39,7 @@ def s3_export_many( for block_number in range(after_block, before_block): if latest_s3_block is not None: if block_number > latest_s3_block: - filepath = f"{filepath_base}" + f"{block_number}" - uri = _get_uri(db_session, bucket, filepath, region) + uri = base_uri + f"/{block_number}" db_session.execute( """ SELECT * FROM aws_s3.query_export_to_s3( @@ -60,19 +55,3 @@ def s3_export_many( }, ) update_latest_s3_block(db_session, block_number) - - -def _get_uri(db_session, bucket: str, filepath: str, region: str) -> str: - - uri = db_session.execute( - """ - SELECT aws_commons.create_s3_uri( - '{bucket}', - '{filepath}', - '{region}' - ) AS s3_uri_1 \gset - """, - params={"bucket": bucket, "filepath": filepath, "region": region}, - ) - - return uri