From 053c29cf209876a3f7a7a81d81365cd205939411 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Wed, 17 Nov 2021 21:59:12 -0500 Subject: [PATCH 1/8] Add placeholder for price commands --- cli.py | 6 ++++++ mev | 16 ++++++++++++++++ pyproject.toml | 1 + 3 files changed, 23 insertions(+) diff --git a/cli.py b/cli.py index 12d62c7..b1c0acf 100644 --- a/cli.py +++ b/cli.py @@ -79,6 +79,12 @@ async def inspect_many_blocks_command( ) +@cli.command() +@coro +async def fetch_all_prices(): + print("fetching") + + def get_rpc_url() -> str: return os.environ["RPC_URL"] diff --git a/mev b/mev index 7d7f79f..69e29c4 100755 --- a/mev +++ b/mev @@ -56,6 +56,22 @@ case "$1" in echo "Fetching block $block_number" kubectl exec -ti deploy/mev-inspect -- poetry run fetch-block $block_number ;; + prices) + shift + case "$1" in + fetch-latest) + echo "running fetch-latest" + ;; + fetch-all) + echo "running fetch-all" + kubectl exec -ti deploy/mev-inspect -- \ + poetry run fetch-all-prices + ;; + *) + echo "prices usage: "$1" {fetch-latest|fetch-all}" + exit 1 + esac + ;; exec) shift kubectl exec -ti deploy/mev-inspect -- $@ diff --git a/pyproject.toml b/pyproject.toml index 9217449..5d69c1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ build-backend = "poetry.core.masonry.api" inspect-block = 'cli:inspect_block_command' inspect-many-blocks = 'cli:inspect_many_blocks_command' fetch-block = 'cli:fetch_block_command' +fetch-all-prices = 'cli:fetch_all_prices' [tool.black] exclude = ''' From 2dc14218bfc102a2397b2f0326f4c6f881c36c7d Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Wed, 17 Nov 2021 23:07:25 -0500 Subject: [PATCH 2/8] Add support for fetching all supported prices --- cli.py | 3 +++ mev_inspect/coinbase.py | 25 +++++++++++++++++++++++++ mev_inspect/prices.py | 29 +++++++++++++++++++++++++++++ mev_inspect/schemas/coinbase.py | 20 ++++++++++++++++++++ mev_inspect/schemas/prices.py | 7 +++++++ mev_inspect/schemas/utils.py | 2 +- 6 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 mev_inspect/coinbase.py create mode 100644 mev_inspect/prices.py create mode 100644 mev_inspect/schemas/coinbase.py create mode 100644 mev_inspect/schemas/prices.py diff --git a/cli.py b/cli.py index b1c0acf..5e04f6a 100644 --- a/cli.py +++ b/cli.py @@ -7,6 +7,7 @@ import click from mev_inspect.concurrency import coro from mev_inspect.db import get_inspect_session, get_trace_session from mev_inspect.inspector import MEVInspector +from mev_inspect.prices import fetch_all_supported_prices RPC_URL_ENV = "RPC_URL" @@ -83,6 +84,8 @@ async def inspect_many_blocks_command( @coro async def fetch_all_prices(): print("fetching") + prices = await fetch_all_supported_prices() + print(prices[0]) def get_rpc_url() -> str: diff --git a/mev_inspect/coinbase.py b/mev_inspect/coinbase.py new file mode 100644 index 0000000..857288b --- /dev/null +++ b/mev_inspect/coinbase.py @@ -0,0 +1,25 @@ +import aiohttp + +from mev_inspect.classifiers.specs.weth import WETH_ADDRESS +from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS +from mev_inspect.schemas.coinbase import CoinbasePrices, CoinbasePricesResponse + + +COINBASE_API_BASE = "https://www.coinbase.com/api/v2" +COINBASE_TOKEN_NAMES = { + WETH_ADDRESS: "weth", + ETH_TOKEN_ADDRESS: "ethereum", +} + + +async def fetch_coinbase_prices(token_address: str) -> CoinbasePrices: + if token_address not in COINBASE_TOKEN_NAMES: + raise ValueError(f"Unsupported token_address {token_address}") + + coinbase_token_name = COINBASE_TOKEN_NAMES[token_address] + url = f"{COINBASE_API_BASE}/assets/prices/{coinbase_token_name}" + + async with aiohttp.ClientSession() as session: + async with session.get(url, params={"base": "USD"}) as response: + json_data = await response.json() + return CoinbasePricesResponse(**json_data).data.prices diff --git a/mev_inspect/prices.py b/mev_inspect/prices.py new file mode 100644 index 0000000..8f0ac4a --- /dev/null +++ b/mev_inspect/prices.py @@ -0,0 +1,29 @@ +from typing import List + +from mev_inspect.classifiers.specs.weth import WETH_ADDRESS +from mev_inspect.coinbase import fetch_coinbase_prices +from mev_inspect.schemas.prices import Price +from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS + + +SUPPORTED_TOKENS = [ + WETH_ADDRESS, + ETH_TOKEN_ADDRESS, +] + + +async def fetch_all_supported_prices() -> List[Price]: + prices = [] + + for token_address in SUPPORTED_TOKENS: + coinbase_prices = await fetch_coinbase_prices(token_address) + for usd_price, timestamp_seconds in coinbase_prices.all.prices: + price = Price( + token_address=token_address, + usd_price=usd_price, + timestamp_seconds=timestamp_seconds, + ) + + prices.append(price) + + return prices diff --git a/mev_inspect/schemas/coinbase.py b/mev_inspect/schemas/coinbase.py new file mode 100644 index 0000000..fca7bab --- /dev/null +++ b/mev_inspect/schemas/coinbase.py @@ -0,0 +1,20 @@ +from typing import List, Tuple + +from pydantic import BaseModel + + +class CoinbasePricesEntry(BaseModel): + # tuple of price and timestamp + prices: List[Tuple[float, int]] + + +class CoinbasePrices(BaseModel): + all: CoinbasePricesEntry + + +class CoinbasePricesDataResponse(BaseModel): + prices: CoinbasePrices + + +class CoinbasePricesResponse(BaseModel): + data: CoinbasePricesDataResponse diff --git a/mev_inspect/schemas/prices.py b/mev_inspect/schemas/prices.py new file mode 100644 index 0000000..55abe0f --- /dev/null +++ b/mev_inspect/schemas/prices.py @@ -0,0 +1,7 @@ +from pydantic import BaseModel + + +class Price(BaseModel): + token_address: str + timestamp_seconds: int + usd_price: float diff --git a/mev_inspect/schemas/utils.py b/mev_inspect/schemas/utils.py index e3b53f6..1d15876 100644 --- a/mev_inspect/schemas/utils.py +++ b/mev_inspect/schemas/utils.py @@ -1,8 +1,8 @@ import json from hexbytes import HexBytes -from web3.datastructures import AttributeDict from pydantic import BaseModel +from web3.datastructures import AttributeDict def to_camel(string: str) -> str: From bed8520bc88728fa829c4e6cc6c83be8dc30b9dd Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 18 Nov 2021 11:55:42 -0500 Subject: [PATCH 3/8] Write prices on fetch-all --- cli.py | 9 +++++++-- mev_inspect/prices.py | 2 +- mev_inspect/schemas/prices.py | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cli.py b/cli.py index 5e04f6a..fd3a30e 100644 --- a/cli.py +++ b/cli.py @@ -5,6 +5,7 @@ import sys import click from mev_inspect.concurrency import coro +from mev_inspect.crud.prices import write_prices from mev_inspect.db import get_inspect_session, get_trace_session from mev_inspect.inspector import MEVInspector from mev_inspect.prices import fetch_all_supported_prices @@ -83,9 +84,13 @@ async def inspect_many_blocks_command( @cli.command() @coro async def fetch_all_prices(): - print("fetching") + inspect_db_session = get_inspect_session() + + print("Fetching prices") prices = await fetch_all_supported_prices() - print(prices[0]) + + print("Writing prices") + write_prices(inspect_db_session, prices) def get_rpc_url() -> str: diff --git a/mev_inspect/prices.py b/mev_inspect/prices.py index 8f0ac4a..8abe23d 100644 --- a/mev_inspect/prices.py +++ b/mev_inspect/prices.py @@ -21,7 +21,7 @@ async def fetch_all_supported_prices() -> List[Price]: price = Price( token_address=token_address, usd_price=usd_price, - timestamp_seconds=timestamp_seconds, + timestamp=timestamp_seconds, ) prices.append(price) diff --git a/mev_inspect/schemas/prices.py b/mev_inspect/schemas/prices.py index 55abe0f..40e5c48 100644 --- a/mev_inspect/schemas/prices.py +++ b/mev_inspect/schemas/prices.py @@ -1,7 +1,9 @@ +from datetime import datetime + from pydantic import BaseModel class Price(BaseModel): token_address: str - timestamp_seconds: int + timestamp: datetime usd_price: float From 5b59427d4f8b34f69a42394016391a71d2180f85 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 18 Nov 2021 13:43:21 -0500 Subject: [PATCH 4/8] Write prices. Ignore duplicates --- mev_inspect/crud/prices.py | 17 +++++++++++++++++ mev_inspect/models/prices.py | 11 +++++++++++ 2 files changed, 28 insertions(+) create mode 100644 mev_inspect/crud/prices.py create mode 100644 mev_inspect/models/prices.py diff --git a/mev_inspect/crud/prices.py b/mev_inspect/crud/prices.py new file mode 100644 index 0000000..97f4166 --- /dev/null +++ b/mev_inspect/crud/prices.py @@ -0,0 +1,17 @@ +from typing import List + +from sqlalchemy.dialects.postgresql import insert + +from mev_inspect.models.prices import PriceModel +from mev_inspect.schemas.prices import Price + + +def write_prices(db_session, prices: List[Price]) -> None: + insert_statement = ( + insert(PriceModel.__table__) + .values([price.dict() for price in prices]) + .on_conflict_do_nothing() + ) + + db_session.execute(insert_statement) + db_session.commit() diff --git a/mev_inspect/models/prices.py b/mev_inspect/models/prices.py new file mode 100644 index 0000000..cb41c83 --- /dev/null +++ b/mev_inspect/models/prices.py @@ -0,0 +1,11 @@ +from sqlalchemy import Column, Numeric, String, TIMESTAMP + +from .base import Base + + +class PriceModel(Base): + __tablename__ = "usd_prices" + + timestamp = Column(TIMESTAMP, nullable=False, primary_key=True) + usd_price = Column(Numeric, nullable=False) + token_address = Column(String, nullable=False, primary_key=True) From d499983f32687a2d73c2f5880f2efd75b505fc70 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 18 Nov 2021 13:45:25 -0500 Subject: [PATCH 5/8] Remove fetch-latest for now --- mev | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mev b/mev index 69e29c4..5893782 100755 --- a/mev +++ b/mev @@ -59,16 +59,13 @@ case "$1" in prices) shift case "$1" in - fetch-latest) - echo "running fetch-latest" - ;; fetch-all) echo "running fetch-all" kubectl exec -ti deploy/mev-inspect -- \ poetry run fetch-all-prices ;; *) - echo "prices usage: "$1" {fetch-latest|fetch-all}" + echo "prices usage: "$1" {fetch-all}" exit 1 esac ;; From 023205c25b60cdc2b0929952f14e181431d96b23 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 18 Nov 2021 13:47:59 -0500 Subject: [PATCH 6/8] Print => logger --- cli.py | 5 +++-- mev | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cli.py b/cli.py index fd3a30e..2a78b75 100644 --- a/cli.py +++ b/cli.py @@ -13,6 +13,7 @@ from mev_inspect.prices import fetch_all_supported_prices RPC_URL_ENV = "RPC_URL" logging.basicConfig(stream=sys.stdout, level=logging.INFO) +logger = logging.getLogger(__name__) @click.group() @@ -86,10 +87,10 @@ async def inspect_many_blocks_command( async def fetch_all_prices(): inspect_db_session = get_inspect_session() - print("Fetching prices") + logger.info("Fetching prices") prices = await fetch_all_supported_prices() - print("Writing prices") + logger.info("Writing prices") write_prices(inspect_db_session, prices) diff --git a/mev b/mev index 5893782..e326f77 100755 --- a/mev +++ b/mev @@ -60,7 +60,7 @@ case "$1" in shift case "$1" in fetch-all) - echo "running fetch-all" + echo "Running price fetch-all" kubectl exec -ti deploy/mev-inspect -- \ poetry run fetch-all-prices ;; From f5233a17fd2e6c6abb7f0cd3104354435996ed21 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Thu, 18 Nov 2021 13:56:07 -0500 Subject: [PATCH 7/8] Rename to prices table --- mev_inspect/models/prices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mev_inspect/models/prices.py b/mev_inspect/models/prices.py index cb41c83..86bf3e0 100644 --- a/mev_inspect/models/prices.py +++ b/mev_inspect/models/prices.py @@ -4,7 +4,7 @@ from .base import Base class PriceModel(Base): - __tablename__ = "usd_prices" + __tablename__ = "prices" timestamp = Column(TIMESTAMP, nullable=False, primary_key=True) usd_price = Column(Numeric, nullable=False) From 4f34316afb7554082822dbe54dda7ff98d8610f1 Mon Sep 17 00:00:00 2001 From: Luke Van Seters Date: Fri, 26 Nov 2021 21:03:57 -0500 Subject: [PATCH 8/8] COINBASE_TOKEN_NAMES => COINBASE_TOKEN_NAME_BY_ADDRESS --- mev_inspect/coinbase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mev_inspect/coinbase.py b/mev_inspect/coinbase.py index 857288b..215a379 100644 --- a/mev_inspect/coinbase.py +++ b/mev_inspect/coinbase.py @@ -6,17 +6,17 @@ from mev_inspect.schemas.coinbase import CoinbasePrices, CoinbasePricesResponse COINBASE_API_BASE = "https://www.coinbase.com/api/v2" -COINBASE_TOKEN_NAMES = { +COINBASE_TOKEN_NAME_BY_ADDRESS = { WETH_ADDRESS: "weth", ETH_TOKEN_ADDRESS: "ethereum", } async def fetch_coinbase_prices(token_address: str) -> CoinbasePrices: - if token_address not in COINBASE_TOKEN_NAMES: + if token_address not in COINBASE_TOKEN_NAME_BY_ADDRESS: raise ValueError(f"Unsupported token_address {token_address}") - coinbase_token_name = COINBASE_TOKEN_NAMES[token_address] + coinbase_token_name = COINBASE_TOKEN_NAME_BY_ADDRESS[token_address] url = f"{COINBASE_API_BASE}/assets/prices/{coinbase_token_name}" async with aiohttp.ClientSession() as session: