Merge pull request #231 from flashbots/coingecko-api

Add coingecko api

cli.py

@@ -8,7 +8,7 @@ from mev_inspect.concurrency import coro
 from mev_inspect.crud.prices import write_prices
 from mev_inspect.db import get_inspect_session, get_trace_session
 from mev_inspect.inspector import MEVInspector
-from mev_inspect.prices import fetch_all_supported_prices
+from mev_inspect.prices import fetch_prices
 
 RPC_URL_ENV = "RPC_URL"
 
@@ -107,12 +107,11 @@ def enqueue_many_blocks_command(after_block: int, before_block: int, batch_size:
 
 
 @cli.command()
-@coro
+def fetch_all_prices():
-async def fetch_all_prices():
     inspect_db_session = get_inspect_session()
 
     logger.info("Fetching prices")
-    prices = await fetch_all_supported_prices()
+    prices = fetch_prices()
 
     logger.info("Writing prices")
     write_prices(inspect_db_session, prices)

mev_inspect/coinbase.py

@@ -1,46 +0,0 @@
-import aiohttp
-
-from mev_inspect.classifiers.specs.weth import WETH_ADDRESS
-from mev_inspect.schemas.coinbase import CoinbasePrices, CoinbasePricesResponse
-from mev_inspect.schemas.prices import (
-    AAVE_TOKEN_ADDRESS,
-    CDAI_TOKEN_ADDRESS,
-    CUSDC_TOKEN_ADDRESS,
-    DAI_TOKEN_ADDRESS,
-    LINK_TOKEN_ADDRESS,
-    REN_TOKEN_ADDRESS,
-    UNI_TOKEN_ADDRESS,
-    USDC_TOKEN_ADDRESS,
-    WBTC_TOKEN_ADDRESS,
-    YEARN_TOKEN_ADDRESS,
-)
-from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS
-
-COINBASE_API_BASE = "https://www.coinbase.com/api/v2"
-COINBASE_TOKEN_NAME_BY_ADDRESS = {
-    WETH_ADDRESS: "weth",
-    ETH_TOKEN_ADDRESS: "ethereum",
-    WBTC_TOKEN_ADDRESS: "wrapped-bitcoin",
-    LINK_TOKEN_ADDRESS: "chainlink",
-    YEARN_TOKEN_ADDRESS: "yearn-finance",
-    AAVE_TOKEN_ADDRESS: "aave",
-    UNI_TOKEN_ADDRESS: "uniswap",
-    USDC_TOKEN_ADDRESS: "usdc",
-    DAI_TOKEN_ADDRESS: "dai",
-    REN_TOKEN_ADDRESS: "ren",
-    CUSDC_TOKEN_ADDRESS: "compound-usd-coin",
-    CDAI_TOKEN_ADDRESS: "compound-dai",
-}
-
-
-async def fetch_coinbase_prices(token_address: str) -> CoinbasePrices:
-    if token_address not in COINBASE_TOKEN_NAME_BY_ADDRESS:
-        raise ValueError(f"Unsupported token_address {token_address}")
-
-    coinbase_token_name = COINBASE_TOKEN_NAME_BY_ADDRESS[token_address]
-    url = f"{COINBASE_API_BASE}/assets/prices/{coinbase_token_name}"
-
-    async with aiohttp.ClientSession() as session:
-        async with session.get(url, params={"base": "USD"}) as response:
-            json_data = await response.json()
-            return CoinbasePricesResponse(**json_data).data.prices

mev_inspect/prices.py

@@ -1,50 +1,33 @@
+from datetime import datetime as dt
 from typing import List
 
-from mev_inspect.classifiers.specs.weth import WETH_ADDRESS
+from pycoingecko import CoinGeckoAPI
-from mev_inspect.coinbase import fetch_coinbase_prices
-from mev_inspect.schemas.prices import (
-    AAVE_TOKEN_ADDRESS,
-    CDAI_TOKEN_ADDRESS,
-    CUSDC_TOKEN_ADDRESS,
-    DAI_TOKEN_ADDRESS,
-    LINK_TOKEN_ADDRESS,
-    REN_TOKEN_ADDRESS,
-    UNI_TOKEN_ADDRESS,
-    USDC_TOKEN_ADDRESS,
-    WBTC_TOKEN_ADDRESS,
-    YEARN_TOKEN_ADDRESS,
-    Price,
-)
-from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS
 
-SUPPORTED_TOKENS = [
+from mev_inspect.schemas.prices import COINGECKO_ID_BY_ADDRESS, TOKEN_ADDRESSES, Price
-    AAVE_TOKEN_ADDRESS,
-    CDAI_TOKEN_ADDRESS,
-    CUSDC_TOKEN_ADDRESS,
-    DAI_TOKEN_ADDRESS,
-    ETH_TOKEN_ADDRESS,
-    LINK_TOKEN_ADDRESS,
-    REN_TOKEN_ADDRESS,
-    UNI_TOKEN_ADDRESS,
-    USDC_TOKEN_ADDRESS,
-    WBTC_TOKEN_ADDRESS,
-    WETH_ADDRESS,
-    YEARN_TOKEN_ADDRESS,
-]
 
 
-async def fetch_all_supported_prices() -> List[Price]:
+def fetch_prices() -> List[Price]:
+    cg = CoinGeckoAPI()
     prices = []
 
-    for token_address in SUPPORTED_TOKENS:
+    for token_address in TOKEN_ADDRESSES:
-        coinbase_prices = await fetch_coinbase_prices(token_address)
+        price_data = cg.get_coin_market_chart_by_id(
-        for usd_price, timestamp_seconds in coinbase_prices.all.prices:
+            id=COINGECKO_ID_BY_ADDRESS[token_address],
-            price = Price(
+            vs_currency="usd",
-                token_address=token_address,
+            days="max",
-                usd_price=usd_price,
+            interval="daily",
-                timestamp=timestamp_seconds,
+        )
+        price_time_series = price_data["prices"]
+
+        for entry in price_time_series:
+            timestamp = dt.fromtimestamp(entry[0] / 100)
+            token_price = entry[1]
+            prices.append(
+                Price(
+                    timestamp=timestamp,
+                    usd_price=token_price,
+                    token_address=token_address,
+                )
             )
 
-            prices.append(price)
-
     return prices

mev_inspect/schemas/coinbase.py

@@ -1,20 +0,0 @@
-from typing import List, Tuple
-
-from pydantic import BaseModel
-
-
-class CoinbasePricesEntry(BaseModel):
-    # tuple of price and timestamp
-    prices: List[Tuple[float, int]]
-
-
-class CoinbasePrices(BaseModel):
-    all: CoinbasePricesEntry
-
-
-class CoinbasePricesDataResponse(BaseModel):
-    prices: CoinbasePrices
-
-
-class CoinbasePricesResponse(BaseModel):
-    data: CoinbasePricesDataResponse

mev_inspect/schemas/prices.py

@@ -2,6 +2,9 @@ from datetime import datetime
 
 from pydantic import BaseModel, validator
 
+from mev_inspect.classifiers.specs.weth import WETH_ADDRESS
+from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS
+
 WBTC_TOKEN_ADDRESS = "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599"
 LINK_TOKEN_ADDRESS = "0x514910771af9ca656af840dff83e8264ecf986ca"
 YEARN_TOKEN_ADDRESS = "0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e"
@@ -12,12 +15,48 @@ DAI_TOKEN_ADDRESS = "0x6b175474e89094c44da98b954eedeac495271d0f"
 REN_TOKEN_ADDRESS = "0x408e41876cccdc0f92210600ef50372656052a38"
 CUSDC_TOKEN_ADDRESS = "0x39aa39c021dfbae8fac545936693ac917d5e7563"
 CDAI_TOKEN_ADDRESS = "0x5d3a536e4d6dbd6114cc1ead35777bab948e3643"
+CETH_TOKEN_ADDRESS = "0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5"
+CWBTC_TOKEN_ADDRESS = "0xc11b1268c1a384e55c48c2391d8d480264a3a7f4"
+
+TOKEN_ADDRESSES = [
+    ETH_TOKEN_ADDRESS,
+    WETH_ADDRESS,
+    WBTC_TOKEN_ADDRESS,
+    LINK_TOKEN_ADDRESS,
+    YEARN_TOKEN_ADDRESS,
+    AAVE_TOKEN_ADDRESS,
+    UNI_TOKEN_ADDRESS,
+    USDC_TOKEN_ADDRESS,
+    DAI_TOKEN_ADDRESS,
+    REN_TOKEN_ADDRESS,
+    CUSDC_TOKEN_ADDRESS,
+    CDAI_TOKEN_ADDRESS,
+    CETH_TOKEN_ADDRESS,
+    CWBTC_TOKEN_ADDRESS,
+]
+
+COINGECKO_ID_BY_ADDRESS = {
+    WETH_ADDRESS: "weth",
+    ETH_TOKEN_ADDRESS: "ethereum",
+    WBTC_TOKEN_ADDRESS: "wrapped-bitcoin",
+    LINK_TOKEN_ADDRESS: "chainlink",
+    YEARN_TOKEN_ADDRESS: "yearn-finance",
+    AAVE_TOKEN_ADDRESS: "aave",
+    UNI_TOKEN_ADDRESS: "uniswap",
+    USDC_TOKEN_ADDRESS: "usd-coin",
+    DAI_TOKEN_ADDRESS: "dai",
+    REN_TOKEN_ADDRESS: "republic-protocol",
+    CUSDC_TOKEN_ADDRESS: "compound-usd-coin",
+    CDAI_TOKEN_ADDRESS: "cdai",
+    CETH_TOKEN_ADDRESS: "compound-ether",
+    CWBTC_TOKEN_ADDRESS: "compound-wrapped-btc",
+}
 
 
 class Price(BaseModel):
     token_address: str
-    timestamp: datetime
     usd_price: float
+    timestamp: datetime
 
     @validator("token_address")
     def lower_token_address(cls, v: str) -> str:

poetry.lock

@@ -730,6 +730,17 @@ category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 
+[[package]]
+name = "pycoingecko"
+version = "2.2.0"
+description = "Python wrapper around the CoinGecko API"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+requests = "*"
+
 [[package]]
 name = "pycryptodome"
 version = "3.10.1"
@@ -1116,7 +1127,7 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.9"
-content-hash = "2ce3bdeb2d8bd31210026e5054a54c67fc766cdf22dc83485eca425643cdf760"
+content-hash = "955c3df01b275e9b4807190e468a2df4d3d18b6a45a7c1659599ef476b35be51"
 
 [metadata.files]
 aiohttp = [
@@ -1775,6 +1786,10 @@ py = [
     {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"},
     {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"},
 ]
+pycoingecko = [
+    {file = "pycoingecko-2.2.0-py3-none-any.whl", hash = "sha256:3646968c8c6936ca4e94b5f562328a763c12a0e9644141cb0215089dda59fe01"},
+    {file = "pycoingecko-2.2.0.tar.gz", hash = "sha256:9add73085729b1f10f93c7948490b09e8cd47c29bebe47dccb319e8b49502d0c"},
+]
 pycryptodome = [
     {file = "pycryptodome-3.10.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1c5e1ca507de2ad93474be5cfe2bfa76b7cf039a1a32fc196f40935944871a06"},
     {file = "pycryptodome-3.10.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:6260e24d41149268122dd39d4ebd5941e9d107f49463f7e071fd397e29923b0c"},

pyproject.toml

@@ -13,6 +13,7 @@ click = "^8.0.1"
 psycopg2 = "^2.9.1"
 aiohttp = "^3.8.0"
 dramatiq = {extras = ["redis"], version = "^1.12.1"}
+pycoingecko = "^2.2.0"
 
 [tool.poetry.dev-dependencies]
 pre-commit = "^2.13.0"