108 lines
2.7 KiB
Python
108 lines
2.7 KiB
Python
import json
|
|
import logging
|
|
import os
|
|
from typing import Optional
|
|
|
|
import boto3
|
|
|
|
from mev_inspect.text_io import BytesIteratorIO
|
|
|
|
AWS_ENDPOINT_URL_ENV = "AWS_ENDPOINT_URL"
|
|
MEV_SUMMARY_EXPORT_BLOCK_QUERY = """
|
|
SELECT to_json(mev_summary)
|
|
FROM mev_summary
|
|
WHERE block_number=:block_number
|
|
"""
|
|
MEV_SUMMARY_EXPORT_RANGE_QUERY = """
|
|
SELECT to_json(mev_summary)
|
|
FROM mev_summary
|
|
WHERE
|
|
block_number >= :after_block_number AND
|
|
block_number < :before_block_number
|
|
"""
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def export_block_range(
|
|
inspect_db_session, after_block_number: int, before_block_number
|
|
) -> None:
|
|
export_bucket_name = get_export_bucket_name()
|
|
client = get_s3_client()
|
|
|
|
mev_summary_json_results = inspect_db_session.execute(
|
|
statement=MEV_SUMMARY_EXPORT_RANGE_QUERY,
|
|
params={
|
|
"after_block_number": after_block_number,
|
|
"before_block_number": before_block_number,
|
|
},
|
|
)
|
|
|
|
mev_summary_json_fileobj = BytesIteratorIO(
|
|
(f"{json.dumps(row)}\n".encode("utf-8") for (row,) in mev_summary_json_results)
|
|
)
|
|
|
|
key = f"mev_summary/flashbots_{after_block_number}_{before_block_number}.json"
|
|
|
|
client.upload_fileobj(
|
|
mev_summary_json_fileobj,
|
|
Bucket=export_bucket_name,
|
|
Key=key,
|
|
)
|
|
|
|
logger.info(f"Exported to {key}")
|
|
|
|
|
|
def export_block(inspect_db_session, block_number: int) -> None:
|
|
export_bucket_name = get_export_bucket_name()
|
|
client = get_s3_client()
|
|
|
|
mev_summary_json_results = inspect_db_session.execute(
|
|
statement=MEV_SUMMARY_EXPORT_BLOCK_QUERY,
|
|
params={"block_number": block_number},
|
|
)
|
|
mev_summary_json_fileobj = BytesIteratorIO(
|
|
(f"{json.dumps(row)}\n".encode("utf-8") for (row,) in mev_summary_json_results)
|
|
)
|
|
|
|
key = f"mev_summary/flashbots_{block_number}.json"
|
|
|
|
client.upload_fileobj(
|
|
mev_summary_json_fileobj,
|
|
Bucket=export_bucket_name,
|
|
Key=key,
|
|
)
|
|
|
|
logger.info(f"Exported to {key}")
|
|
|
|
|
|
def get_s3_client():
|
|
endpoint_url = get_endpoint_url()
|
|
return boto3.client(
|
|
"s3",
|
|
endpoint_url=endpoint_url,
|
|
region_name=get_export_bucket_region(),
|
|
aws_access_key_id=get_export_aws_access_key_id(),
|
|
aws_secret_access_key=get_export_aws_secret_access_key(),
|
|
)
|
|
|
|
|
|
def get_endpoint_url() -> Optional[str]:
|
|
return os.environ.get(AWS_ENDPOINT_URL_ENV)
|
|
|
|
|
|
def get_export_bucket_name() -> str:
|
|
return os.environ["EXPORT_BUCKET_NAME"]
|
|
|
|
|
|
def get_export_bucket_region() -> str:
|
|
return os.environ["EXPORT_BUCKET_REGION"]
|
|
|
|
|
|
def get_export_aws_access_key_id() -> str:
|
|
return os.environ["EXPORT_AWS_ACCESS_KEY_ID"]
|
|
|
|
|
|
def get_export_aws_secret_access_key() -> str:
|
|
return os.environ["EXPORT_AWS_SECRET_ACCESS_KEY"]
|