Skip write if no data and no key or current upload has no data

This commit is contained in:
Luke Van Seters 2022-02-16 09:06:51 -05:00
parent b30d6be0c5
commit 4ef2145409

View File

@ -27,6 +27,7 @@ logger = logging.getLogger(__name__)
def export_block(inspect_db_session, block_number: int) -> None: def export_block(inspect_db_session, block_number: int) -> None:
export_bucket_name = get_export_bucket_name() export_bucket_name = get_export_bucket_name()
client = get_s3_client() client = get_s3_client()
object_key = f"mev_summary/flashbots_{block_number}.json"
mev_summary_json_results = inspect_db_session.execute( mev_summary_json_results = inspect_db_session.execute(
statement=MEV_SUMMARY_EXPORT_QUERY, statement=MEV_SUMMARY_EXPORT_QUERY,
@ -37,23 +38,35 @@ def export_block(inspect_db_session, block_number: int) -> None:
first_value, mev_summary_json_results = _peek(mev_summary_json_results) first_value, mev_summary_json_results = _peek(mev_summary_json_results)
if first_value is None: if first_value is None:
logger.info("No data for this block") existing_object_size = _get_object_size(client, export_bucket_name, object_key)
else: if existing_object_size is None or existing_object_size == 0:
logger.info("We have data for this block") logger.info(f"Skipping block {block_number} - no data")
return
mev_summary_json_fileobj = BytesIteratorIO( mev_summary_json_fileobj = BytesIteratorIO(
(f"{json.dumps(row)}\n".encode("utf-8") for (row,) in mev_summary_json_results) (f"{json.dumps(row)}\n".encode("utf-8") for (row,) in mev_summary_json_results)
) )
key = f"mev_summary/flashbots_{block_number}.json"
client.upload_fileobj( client.upload_fileobj(
mev_summary_json_fileobj, mev_summary_json_fileobj,
Bucket=export_bucket_name, Bucket=export_bucket_name,
Key=key, Key=object_key,
) )
logger.info(f"Exported to {key}") logger.info(f"Exported to {object_key}")
def _get_object_size(client, bucket: str, key: str) -> Optional[int]:
response = client.list_objects_v2(
Bucket=bucket,
Prefix=key,
)
for obj in response.get("Contents", []):
if obj["Key"] == key:
return obj["Size"]
return None
def get_s3_client(): def get_s3_client():