diff --git a/cli.py b/cli.py index c9c815f..f67441d 100644 --- a/cli.py +++ b/cli.py @@ -8,7 +8,7 @@ from mev_inspect.concurrency import coro from mev_inspect.crud.prices import write_prices from mev_inspect.db import get_inspect_session, get_trace_session from mev_inspect.inspector import MEVInspector -from mev_inspect.prices import fetch_all_supported_prices +from mev_inspect.prices import fetch_prices RPC_URL_ENV = "RPC_URL" @@ -107,12 +107,11 @@ def enqueue_many_blocks_command(after_block: int, before_block: int, batch_size: @cli.command() -@coro -async def fetch_all_prices(): +def fetch_all_prices(): inspect_db_session = get_inspect_session() logger.info("Fetching prices") - prices = await fetch_all_supported_prices() + prices = fetch_prices() logger.info("Writing prices") write_prices(inspect_db_session, prices) diff --git a/mev_inspect/coinbase.py b/mev_inspect/coinbase.py deleted file mode 100644 index df1996c..0000000 --- a/mev_inspect/coinbase.py +++ /dev/null @@ -1,46 +0,0 @@ -import aiohttp - -from mev_inspect.classifiers.specs.weth import WETH_ADDRESS -from mev_inspect.schemas.coinbase import CoinbasePrices, CoinbasePricesResponse -from mev_inspect.schemas.prices import ( - AAVE_TOKEN_ADDRESS, - CDAI_TOKEN_ADDRESS, - CUSDC_TOKEN_ADDRESS, - DAI_TOKEN_ADDRESS, - LINK_TOKEN_ADDRESS, - REN_TOKEN_ADDRESS, - UNI_TOKEN_ADDRESS, - USDC_TOKEN_ADDRESS, - WBTC_TOKEN_ADDRESS, - YEARN_TOKEN_ADDRESS, -) -from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS - -COINBASE_API_BASE = "https://www.coinbase.com/api/v2" -COINBASE_TOKEN_NAME_BY_ADDRESS = { - WETH_ADDRESS: "weth", - ETH_TOKEN_ADDRESS: "ethereum", - WBTC_TOKEN_ADDRESS: "wrapped-bitcoin", - LINK_TOKEN_ADDRESS: "chainlink", - YEARN_TOKEN_ADDRESS: "yearn-finance", - AAVE_TOKEN_ADDRESS: "aave", - UNI_TOKEN_ADDRESS: "uniswap", - USDC_TOKEN_ADDRESS: "usdc", - DAI_TOKEN_ADDRESS: "dai", - REN_TOKEN_ADDRESS: "ren", - CUSDC_TOKEN_ADDRESS: "compound-usd-coin", - CDAI_TOKEN_ADDRESS: "compound-dai", -} - - -async def fetch_coinbase_prices(token_address: str) -> CoinbasePrices: - if token_address not in COINBASE_TOKEN_NAME_BY_ADDRESS: - raise ValueError(f"Unsupported token_address {token_address}") - - coinbase_token_name = COINBASE_TOKEN_NAME_BY_ADDRESS[token_address] - url = f"{COINBASE_API_BASE}/assets/prices/{coinbase_token_name}" - - async with aiohttp.ClientSession() as session: - async with session.get(url, params={"base": "USD"}) as response: - json_data = await response.json() - return CoinbasePricesResponse(**json_data).data.prices diff --git a/mev_inspect/prices.py b/mev_inspect/prices.py index 812bfa3..e69394c 100644 --- a/mev_inspect/prices.py +++ b/mev_inspect/prices.py @@ -1,50 +1,33 @@ +from datetime import datetime as dt from typing import List -from mev_inspect.classifiers.specs.weth import WETH_ADDRESS -from mev_inspect.coinbase import fetch_coinbase_prices -from mev_inspect.schemas.prices import ( - AAVE_TOKEN_ADDRESS, - CDAI_TOKEN_ADDRESS, - CUSDC_TOKEN_ADDRESS, - DAI_TOKEN_ADDRESS, - LINK_TOKEN_ADDRESS, - REN_TOKEN_ADDRESS, - UNI_TOKEN_ADDRESS, - USDC_TOKEN_ADDRESS, - WBTC_TOKEN_ADDRESS, - YEARN_TOKEN_ADDRESS, - Price, -) -from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS +from pycoingecko import CoinGeckoAPI -SUPPORTED_TOKENS = [ - AAVE_TOKEN_ADDRESS, - CDAI_TOKEN_ADDRESS, - CUSDC_TOKEN_ADDRESS, - DAI_TOKEN_ADDRESS, - ETH_TOKEN_ADDRESS, - LINK_TOKEN_ADDRESS, - REN_TOKEN_ADDRESS, - UNI_TOKEN_ADDRESS, - USDC_TOKEN_ADDRESS, - WBTC_TOKEN_ADDRESS, - WETH_ADDRESS, - YEARN_TOKEN_ADDRESS, -] +from mev_inspect.schemas.prices import COINGECKO_ID_BY_ADDRESS, TOKEN_ADDRESSES, Price -async def fetch_all_supported_prices() -> List[Price]: +def fetch_prices() -> List[Price]: + cg = CoinGeckoAPI() prices = [] - for token_address in SUPPORTED_TOKENS: - coinbase_prices = await fetch_coinbase_prices(token_address) - for usd_price, timestamp_seconds in coinbase_prices.all.prices: - price = Price( - token_address=token_address, - usd_price=usd_price, - timestamp=timestamp_seconds, + for token_address in TOKEN_ADDRESSES: + price_data = cg.get_coin_market_chart_by_id( + id=COINGECKO_ID_BY_ADDRESS[token_address], + vs_currency="usd", + days="max", + interval="daily", + ) + price_time_series = price_data["prices"] + + for entry in price_time_series: + timestamp = dt.fromtimestamp(entry[0] / 100) + token_price = entry[1] + prices.append( + Price( + timestamp=timestamp, + usd_price=token_price, + token_address=token_address, + ) ) - prices.append(price) - return prices diff --git a/mev_inspect/schemas/coinbase.py b/mev_inspect/schemas/coinbase.py deleted file mode 100644 index fca7bab..0000000 --- a/mev_inspect/schemas/coinbase.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import List, Tuple - -from pydantic import BaseModel - - -class CoinbasePricesEntry(BaseModel): - # tuple of price and timestamp - prices: List[Tuple[float, int]] - - -class CoinbasePrices(BaseModel): - all: CoinbasePricesEntry - - -class CoinbasePricesDataResponse(BaseModel): - prices: CoinbasePrices - - -class CoinbasePricesResponse(BaseModel): - data: CoinbasePricesDataResponse diff --git a/mev_inspect/schemas/prices.py b/mev_inspect/schemas/prices.py index 3c7f5fd..b04000f 100644 --- a/mev_inspect/schemas/prices.py +++ b/mev_inspect/schemas/prices.py @@ -2,6 +2,9 @@ from datetime import datetime from pydantic import BaseModel, validator +from mev_inspect.classifiers.specs.weth import WETH_ADDRESS +from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS + WBTC_TOKEN_ADDRESS = "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599" LINK_TOKEN_ADDRESS = "0x514910771af9ca656af840dff83e8264ecf986ca" YEARN_TOKEN_ADDRESS = "0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e" @@ -12,12 +15,48 @@ DAI_TOKEN_ADDRESS = "0x6b175474e89094c44da98b954eedeac495271d0f" REN_TOKEN_ADDRESS = "0x408e41876cccdc0f92210600ef50372656052a38" CUSDC_TOKEN_ADDRESS = "0x39aa39c021dfbae8fac545936693ac917d5e7563" CDAI_TOKEN_ADDRESS = "0x5d3a536e4d6dbd6114cc1ead35777bab948e3643" +CETH_TOKEN_ADDRESS = "0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5" +CWBTC_TOKEN_ADDRESS = "0xc11b1268c1a384e55c48c2391d8d480264a3a7f4" + +TOKEN_ADDRESSES = [ + ETH_TOKEN_ADDRESS, + WETH_ADDRESS, + WBTC_TOKEN_ADDRESS, + LINK_TOKEN_ADDRESS, + YEARN_TOKEN_ADDRESS, + AAVE_TOKEN_ADDRESS, + UNI_TOKEN_ADDRESS, + USDC_TOKEN_ADDRESS, + DAI_TOKEN_ADDRESS, + REN_TOKEN_ADDRESS, + CUSDC_TOKEN_ADDRESS, + CDAI_TOKEN_ADDRESS, + CETH_TOKEN_ADDRESS, + CWBTC_TOKEN_ADDRESS, +] + +COINGECKO_ID_BY_ADDRESS = { + WETH_ADDRESS: "weth", + ETH_TOKEN_ADDRESS: "ethereum", + WBTC_TOKEN_ADDRESS: "wrapped-bitcoin", + LINK_TOKEN_ADDRESS: "chainlink", + YEARN_TOKEN_ADDRESS: "yearn-finance", + AAVE_TOKEN_ADDRESS: "aave", + UNI_TOKEN_ADDRESS: "uniswap", + USDC_TOKEN_ADDRESS: "usd-coin", + DAI_TOKEN_ADDRESS: "dai", + REN_TOKEN_ADDRESS: "republic-protocol", + CUSDC_TOKEN_ADDRESS: "compound-usd-coin", + CDAI_TOKEN_ADDRESS: "cdai", + CETH_TOKEN_ADDRESS: "compound-ether", + CWBTC_TOKEN_ADDRESS: "compound-wrapped-btc", +} class Price(BaseModel): token_address: str - timestamp: datetime usd_price: float + timestamp: datetime @validator("token_address") def lower_token_address(cls, v: str) -> str: diff --git a/poetry.lock b/poetry.lock index 38c29fd..fc708b4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -730,6 +730,17 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "pycoingecko" +version = "2.2.0" +description = "Python wrapper around the CoinGecko API" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +requests = "*" + [[package]] name = "pycryptodome" version = "3.10.1" @@ -1116,7 +1127,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "2ce3bdeb2d8bd31210026e5054a54c67fc766cdf22dc83485eca425643cdf760" +content-hash = "955c3df01b275e9b4807190e468a2df4d3d18b6a45a7c1659599ef476b35be51" [metadata.files] aiohttp = [ @@ -1775,6 +1786,10 @@ py = [ {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] +pycoingecko = [ + {file = "pycoingecko-2.2.0-py3-none-any.whl", hash = "sha256:3646968c8c6936ca4e94b5f562328a763c12a0e9644141cb0215089dda59fe01"}, + {file = "pycoingecko-2.2.0.tar.gz", hash = "sha256:9add73085729b1f10f93c7948490b09e8cd47c29bebe47dccb319e8b49502d0c"}, +] pycryptodome = [ {file = "pycryptodome-3.10.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1c5e1ca507de2ad93474be5cfe2bfa76b7cf039a1a32fc196f40935944871a06"}, {file = "pycryptodome-3.10.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:6260e24d41149268122dd39d4ebd5941e9d107f49463f7e071fd397e29923b0c"}, diff --git a/pyproject.toml b/pyproject.toml index 3c4be4d..f2f9f17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ click = "^8.0.1" psycopg2 = "^2.9.1" aiohttp = "^3.8.0" dramatiq = {extras = ["redis"], version = "^1.12.1"} +pycoingecko = "^2.2.0" [tool.poetry.dev-dependencies] pre-commit = "^2.13.0"