Skip write if no data and no key or current upload has no data
This commit is contained in:
parent
b30d6be0c5
commit
4ef2145409
@ -27,6 +27,7 @@ logger = logging.getLogger(__name__)
|
|||||||
def export_block(inspect_db_session, block_number: int) -> None:
|
def export_block(inspect_db_session, block_number: int) -> None:
|
||||||
export_bucket_name = get_export_bucket_name()
|
export_bucket_name = get_export_bucket_name()
|
||||||
client = get_s3_client()
|
client = get_s3_client()
|
||||||
|
object_key = f"mev_summary/flashbots_{block_number}.json"
|
||||||
|
|
||||||
mev_summary_json_results = inspect_db_session.execute(
|
mev_summary_json_results = inspect_db_session.execute(
|
||||||
statement=MEV_SUMMARY_EXPORT_QUERY,
|
statement=MEV_SUMMARY_EXPORT_QUERY,
|
||||||
@ -37,23 +38,35 @@ def export_block(inspect_db_session, block_number: int) -> None:
|
|||||||
|
|
||||||
first_value, mev_summary_json_results = _peek(mev_summary_json_results)
|
first_value, mev_summary_json_results = _peek(mev_summary_json_results)
|
||||||
if first_value is None:
|
if first_value is None:
|
||||||
logger.info("No data for this block")
|
existing_object_size = _get_object_size(client, export_bucket_name, object_key)
|
||||||
else:
|
if existing_object_size is None or existing_object_size == 0:
|
||||||
logger.info("We have data for this block")
|
logger.info(f"Skipping block {block_number} - no data")
|
||||||
|
return
|
||||||
|
|
||||||
mev_summary_json_fileobj = BytesIteratorIO(
|
mev_summary_json_fileobj = BytesIteratorIO(
|
||||||
(f"{json.dumps(row)}\n".encode("utf-8") for (row,) in mev_summary_json_results)
|
(f"{json.dumps(row)}\n".encode("utf-8") for (row,) in mev_summary_json_results)
|
||||||
)
|
)
|
||||||
|
|
||||||
key = f"mev_summary/flashbots_{block_number}.json"
|
|
||||||
|
|
||||||
client.upload_fileobj(
|
client.upload_fileobj(
|
||||||
mev_summary_json_fileobj,
|
mev_summary_json_fileobj,
|
||||||
Bucket=export_bucket_name,
|
Bucket=export_bucket_name,
|
||||||
Key=key,
|
Key=object_key,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info(f"Exported to {key}")
|
logger.info(f"Exported to {object_key}")
|
||||||
|
|
||||||
|
|
||||||
|
def _get_object_size(client, bucket: str, key: str) -> Optional[int]:
|
||||||
|
response = client.list_objects_v2(
|
||||||
|
Bucket=bucket,
|
||||||
|
Prefix=key,
|
||||||
|
)
|
||||||
|
|
||||||
|
for obj in response.get("Contents", []):
|
||||||
|
if obj["Key"] == key:
|
||||||
|
return obj["Size"]
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def get_s3_client():
|
def get_s3_client():
|
||||||
|
Loading…
x
Reference in New Issue
Block a user