From c4f82bdbd644847850ab5d1591ce10b175668b07 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Wed, 19 Jan 2022 15:43:59 -0500 Subject: [PATCH 1/4] Add prices-range --- cli.py | 15 ++++++++++++++- mev | 9 ++++++++- mev_inspect/prices.py | 27 ++++++++++++++++++++++++++- pyproject.toml | 1 + 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/cli.py b/cli.py index f67441d..0729db8 100644 --- a/cli.py +++ b/cli.py @@ -8,7 +8,7 @@ from mev_inspect.concurrency import coro from mev_inspect.crud.prices import write_prices from mev_inspect.db import get_inspect_session, get_trace_session from mev_inspect.inspector import MEVInspector -from mev_inspect.prices import fetch_prices +from mev_inspect.prices import fetch_prices, fetch_prices_range RPC_URL_ENV = "RPC_URL" @@ -117,6 +117,19 @@ def fetch_all_prices(): write_prices(inspect_db_session, prices) +@cli.command() +@click.argument("start", type=int) +@click.argument("end", type=int) +def fetch_range(start: int, end: int): + inspect_db_session = get_inspect_session() + + logger.info("Fetching prices") + prices = fetch_prices_range(start, end) + + logger.info("Writing prices") + write_prices(inspect_db_session, prices) + + def get_rpc_url() -> str: return os.environ["RPC_URL"] diff --git a/mev b/mev index d42e8ee..2505a3b 100755 --- a/mev +++ b/mev @@ -82,7 +82,14 @@ case "$1" in kubectl exec -ti deploy/mev-inspect -- \ poetry run fetch-all-prices ;; - *) + fetch-range) + start=$2 + end=$3 + echo "Running price fetch-range" + kubectl exec -ti deploy/mev-inspect -- \ + poetry run fetch-range $start $end + ;; + *) echo "prices usage: "$1" {fetch-all}" exit 1 esac diff --git a/mev_inspect/prices.py b/mev_inspect/prices.py index e69394c..1f2c3bc 100644 --- a/mev_inspect/prices.py +++ b/mev_inspect/prices.py @@ -20,7 +20,32 @@ def fetch_prices() -> List[Price]: price_time_series = price_data["prices"] for entry in price_time_series: - timestamp = dt.fromtimestamp(entry[0] / 100) + timestamp = dt.fromtimestamp(entry[0] / 1000) + token_price = entry[1] + prices.append( + Price( + timestamp=timestamp, + usd_price=token_price, + token_address=token_address, + ) + ) + + return prices + + +def fetch_prices_range(start: int, end: int) -> List[Price]: + + cg = CoinGeckoAPI() + prices = [] + + for token_address in TOKEN_ADDRESSES: + price_data = cg.get_coin_market_chart_range_by_id( + COINGECKO_ID_BY_ADDRESS[token_address], "usd", start, end + ) + price_time_series = price_data["prices"] + + for entry in price_time_series: + timestamp = dt.fromtimestamp(entry[0] / 1000) token_price = entry[1] prices.append( Price( diff --git a/pyproject.toml b/pyproject.toml index f2f9f17..d616834 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ inspect-many-blocks = 'cli:inspect_many_blocks_command' enqueue-many-blocks = 'cli:enqueue_many_blocks_command' fetch-block = 'cli:fetch_block_command' fetch-all-prices = 'cli:fetch_all_prices' +fetch-range = 'cli:fetch_range' [tool.black] exclude = ''' From f0064e01b27e2b189d3d6bc05b5a0d5feb6f50e8 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Wed, 19 Jan 2022 16:48:20 -0500 Subject: [PATCH 2/4] Shared function, start/end -> after/before --- cli.py | 9 ++++---- mev | 6 ++--- mev_inspect/prices.py | 54 ++++++++++++++++++++----------------------- 3 files changed, 33 insertions(+), 36 deletions(-) diff --git a/cli.py b/cli.py index 0729db8..e771252 100644 --- a/cli.py +++ b/cli.py @@ -1,6 +1,7 @@ import logging import os import sys +from datetime import datetime import click @@ -118,13 +119,13 @@ def fetch_all_prices(): @cli.command() -@click.argument("start", type=int) -@click.argument("end", type=int) -def fetch_range(start: int, end: int): +@click.argument("after", type=datetime) +@click.argument("before", type=datetime) +def fetch_range(after: datetime, before: datetime): inspect_db_session = get_inspect_session() logger.info("Fetching prices") - prices = fetch_prices_range(start, end) + prices = fetch_prices_range(after, before) logger.info("Writing prices") write_prices(inspect_db_session, prices) diff --git a/mev b/mev index 2505a3b..eb81fc6 100755 --- a/mev +++ b/mev @@ -83,11 +83,11 @@ case "$1" in poetry run fetch-all-prices ;; fetch-range) - start=$2 - end=$3 + after=$2 + before=$3 echo "Running price fetch-range" kubectl exec -ti deploy/mev-inspect -- \ - poetry run fetch-range $start $end + poetry run fetch-range $after $before ;; *) echo "prices usage: "$1" {fetch-all}" diff --git a/mev_inspect/prices.py b/mev_inspect/prices.py index 1f2c3bc..8e62169 100644 --- a/mev_inspect/prices.py +++ b/mev_inspect/prices.py @@ -1,4 +1,4 @@ -from datetime import datetime as dt +from datetime import datetime from typing import List from pycoingecko import CoinGeckoAPI @@ -11,48 +11,44 @@ def fetch_prices() -> List[Price]: prices = [] for token_address in TOKEN_ADDRESSES: - price_data = cg.get_coin_market_chart_by_id( + coingecko_price_data = cg.get_coin_market_chart_by_id( id=COINGECKO_ID_BY_ADDRESS[token_address], vs_currency="usd", days="max", interval="daily", ) - price_time_series = price_data["prices"] - - for entry in price_time_series: - timestamp = dt.fromtimestamp(entry[0] / 1000) - token_price = entry[1] - prices.append( - Price( - timestamp=timestamp, - usd_price=token_price, - token_address=token_address, - ) - ) + prices += _build_token_prices(coingecko_price_data, token_address) return prices -def fetch_prices_range(start: int, end: int) -> List[Price]: - +def fetch_prices_range(after: datetime, before: datetime) -> List[Price]: cg = CoinGeckoAPI() prices = [] + after_unix = int(after.timestamp()) + before_unix = int(before.timestamp()) for token_address in TOKEN_ADDRESSES: - price_data = cg.get_coin_market_chart_range_by_id( - COINGECKO_ID_BY_ADDRESS[token_address], "usd", start, end + coingecko_price_data = cg.get_coin_market_chart_range_by_id( + COINGECKO_ID_BY_ADDRESS[token_address], "usd", after_unix, before_unix ) - price_time_series = price_data["prices"] - for entry in price_time_series: - timestamp = dt.fromtimestamp(entry[0] / 1000) - token_price = entry[1] - prices.append( - Price( - timestamp=timestamp, - usd_price=token_price, - token_address=token_address, - ) - ) + prices += _build_token_prices(coingecko_price_data, token_address) return prices + + +def _build_token_prices(coingecko_price_data, token_address) -> List[Price]: + time_series = coingecko_price_data["prices"] + prices = [] + for entry in time_series: + timestamp = datetime.fromtimestamp(entry[0] / 1000) + token_price = entry[1] + prices.append( + Price( + timestamp=timestamp, + usd_price=token_price, + token_address=token_address, + ) + ) + return prices From cdb5ecc9a02eff13914cad39784163a53b79d56a Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Wed, 19 Jan 2022 16:59:25 -0500 Subject: [PATCH 3/4] Fix datetime support --- cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli.py b/cli.py index e771252..e480ea8 100644 --- a/cli.py +++ b/cli.py @@ -119,8 +119,8 @@ def fetch_all_prices(): @cli.command() -@click.argument("after", type=datetime) -@click.argument("before", type=datetime) +@click.argument("after", type=click.DateTime(formats=["%Y-%m-%d", "%m-%d-%Y"])) +@click.argument("before", type=click.DateTime(formats=["%Y-%m-%d", "%m-%d-%Y"])) def fetch_range(after: datetime, before: datetime): inspect_db_session = get_inspect_session() From df8525d582c7e86390cb75434df2c6e91009cdd6 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Wed, 19 Jan 2022 17:03:06 -0500 Subject: [PATCH 4/4] Correct instance name --- mev_inspect/prices.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mev_inspect/prices.py b/mev_inspect/prices.py index 8e62169..62b3508 100644 --- a/mev_inspect/prices.py +++ b/mev_inspect/prices.py @@ -7,11 +7,11 @@ from mev_inspect.schemas.prices import COINGECKO_ID_BY_ADDRESS, TOKEN_ADDRESSES, def fetch_prices() -> List[Price]: - cg = CoinGeckoAPI() + coingecko_api = CoinGeckoAPI() prices = [] for token_address in TOKEN_ADDRESSES: - coingecko_price_data = cg.get_coin_market_chart_by_id( + coingecko_price_data = coingecko_api.get_coin_market_chart_by_id( id=COINGECKO_ID_BY_ADDRESS[token_address], vs_currency="usd", days="max", @@ -23,13 +23,13 @@ def fetch_prices() -> List[Price]: def fetch_prices_range(after: datetime, before: datetime) -> List[Price]: - cg = CoinGeckoAPI() + coingecko_api = CoinGeckoAPI() prices = [] after_unix = int(after.timestamp()) before_unix = int(before.timestamp()) for token_address in TOKEN_ADDRESSES: - coingecko_price_data = cg.get_coin_market_chart_range_by_id( + coingecko_price_data = coingecko_api.get_coin_market_chart_range_by_id( COINGECKO_ID_BY_ADDRESS[token_address], "usd", after_unix, before_unix )