crowetic/mev-inspect-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add uri as parameter

Browse Source
This commit is contained in:
Gui Heise 2022-02-01 14:35:10 -05:00
parent e01de19e73
commit 8496c5ab45
3 changed files with 13 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cli.py
View File

@ -134,14 +134,12 @@ def fetch_range(after: datetime, before: datetime):
@cli.command() @cli.command()
@click.argument("block_number", type=int) @click.argument("block_number", type=int)
@click.argument("bucket", type=str) @click.argument("uri", type=str)
@click.argument("filepath", type=str) def s3_export_command(block_number: int, uri: str):
@click.argument("region", type=str)
def s3_export_command(block_number: int, bucket: str, filepath: str, region: str):
inspect_db_session = get_inspect_session() inspect_db_session = get_inspect_session()
logger.info(f"Exporting block {block_number}") logger.info(f"Exporting block {block_number}")
s3_export(inspect_db_session, block_number, bucket, filepath, region) s3_export(inspect_db_session, uri)
return None return None
@ -149,17 +147,15 @@ def s3_export_command(block_number: int, bucket: str, filepath: str, region: str
@cli.command() @cli.command()
@click.argument("after_block", type=int) @click.argument("after_block", type=int)
@click.argument("before_block", type=int) @click.argument("before_block", type=int)
@click.argument("bucket", type=str) @click.argument("base_uri", type=str)
@click.argument("filepath_base", type=str)
@click.argument("region", type=str)
def s3_export_many_command( def s3_export_many_command(
after_block: int, before_block: int, bucket: str, filepath_base: str, region: str after_block: int, before_block: int, base_uri: str
): ):
inspect_db_session = get_inspect_session() inspect_db_session = get_inspect_session()
logger.info(f"Exporting blocks {after_block} to {before_block}") logger.info(f"Exporting blocks {after_block} to {before_block}")
s3_export_many( s3_export_many(
inspect_db_session, after_block, before_block, bucket, filepath_base, region inspect_db_session, after_block, before_block, base_uri
) )
return None return None

12
mev
View File

@ -99,23 +99,19 @@ case "$1" in
case "$1" in case "$1" in
s3-export) s3-export)
block_number=$2 block_number=$2
bucket=$3 uri=$3
filepath=$4
region=$5
echo "Exporting block" echo "Exporting block"
kubectl exec -ti deploy/mev-inspect -- \ kubectl exec -ti deploy/mev-inspect -- \
poetry run s3-export $block_number $bucket $filepath $region poetry run s3-export $block_number $uri
;; ;;
s3-export-many) s3-export-many)
after_block=$2 after_block=$2
before_block=$3 before_block=$3
bucket=$4 base_uri=$4
filepath_base=$5
region=$6
echo "Exporting blocks" echo "Exporting blocks"
kubectl exec -ti deploy/mev-inspect -- \ kubectl exec -ti deploy/mev-inspect -- \
poetry run s3-export-many $after_block $before_block $bucket $filepath_base $region poetry run s3-export-many $after_block $before_block $base_uri
;; ;;
*) *)
echo "export usage: "$1" {s3-export}" echo "export usage: "$1" {s3-export}"

27
mev_inspect/export.py
View File

@ -5,12 +5,10 @@ from mev_inspect.crud.latest_s3_block import (
def s3_export( def s3_export(
db_session, block_number: int, bucket: str, filepath: str, region: str db_session, block_number: int, uri: str,
) -> None: ) -> None:
"""Export block to S3""" """Export block to S3"""
uri = _get_uri(db_session, bucket, filepath, region)
latest_s3_block = find_latest_s3_block(db_session) latest_s3_block = find_latest_s3_block(db_session)
if latest_s3_block is not None: if latest_s3_block is not None:
@ -32,9 +30,7 @@ def s3_export_many(
db_session, db_session,
after_block: int, after_block: int,
before_block: int, before_block: int,
bucket: str, base_uri: str,
filepath_base: str,
region: str,
) -> None: ) -> None:
"""Export block range to S3""" """Export block range to S3"""
@ -43,8 +39,7 @@ def s3_export_many(
for block_number in range(after_block, before_block): for block_number in range(after_block, before_block):
if latest_s3_block is not None: if latest_s3_block is not None:
if block_number > latest_s3_block: if block_number > latest_s3_block:
filepath = f"{filepath_base}" + f"{block_number}" uri = base_uri + f"/{block_number}"
uri = _get_uri(db_session, bucket, filepath, region)
db_session.execute( db_session.execute(
""" """
SELECT * FROM aws_s3.query_export_to_s3( SELECT * FROM aws_s3.query_export_to_s3(
@ -60,19 +55,3 @@ def s3_export_many(
}, },
) )
update_latest_s3_block(db_session, block_number) update_latest_s3_block(db_session, block_number)
def _get_uri(db_session, bucket: str, filepath: str, region: str) -> str:
uri = db_session.execute(
"""
SELECT aws_commons.create_s3_uri(
'{bucket}',
'{filepath}',
'{region}'
) AS s3_uri_1 \gset
""",
params={"bucket": bucket, "filepath": filepath, "region": region},
)
return uri