From 2e2210371376f480b6093cd66e6b3474a46c29b7 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Thu, 13 Jan 2022 01:26:53 -0500 Subject: [PATCH 1/8] Add coingecko api --- cli.py | 4 +-- mev_inspect/prices.py | 60 +++++++++++---------------------- mev_inspect/schemas/coinbase.py | 20 ----------- mev_inspect/schemas/prices.py | 33 +++++++++++++++++- poetry.lock | 17 +++++++++- pyproject.toml | 1 + 6 files changed, 71 insertions(+), 64 deletions(-) delete mode 100644 mev_inspect/schemas/coinbase.py diff --git a/cli.py b/cli.py index c9c815f..4cbaeb0 100644 --- a/cli.py +++ b/cli.py @@ -8,7 +8,7 @@ from mev_inspect.concurrency import coro from mev_inspect.crud.prices import write_prices from mev_inspect.db import get_inspect_session, get_trace_session from mev_inspect.inspector import MEVInspector -from mev_inspect.prices import fetch_all_supported_prices +from mev_inspect.prices import fetch_prices RPC_URL_ENV = "RPC_URL" @@ -112,7 +112,7 @@ async def fetch_all_prices(): inspect_db_session = get_inspect_session() logger.info("Fetching prices") - prices = await fetch_all_supported_prices() + prices = fetch_prices() logger.info("Writing prices") write_prices(inspect_db_session, prices) diff --git a/mev_inspect/prices.py b/mev_inspect/prices.py index 812bfa3..94816c6 100644 --- a/mev_inspect/prices.py +++ b/mev_inspect/prices.py @@ -1,50 +1,30 @@ +from datetime import datetime as dt from typing import List -from mev_inspect.classifiers.specs.weth import WETH_ADDRESS -from mev_inspect.coinbase import fetch_coinbase_prices -from mev_inspect.schemas.prices import ( - AAVE_TOKEN_ADDRESS, - CDAI_TOKEN_ADDRESS, - CUSDC_TOKEN_ADDRESS, - DAI_TOKEN_ADDRESS, - LINK_TOKEN_ADDRESS, - REN_TOKEN_ADDRESS, - UNI_TOKEN_ADDRESS, - USDC_TOKEN_ADDRESS, - WBTC_TOKEN_ADDRESS, - YEARN_TOKEN_ADDRESS, - Price, -) -from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS +from pycoingecko import CoinGeckoAPI -SUPPORTED_TOKENS = [ - AAVE_TOKEN_ADDRESS, - CDAI_TOKEN_ADDRESS, - CUSDC_TOKEN_ADDRESS, - DAI_TOKEN_ADDRESS, - ETH_TOKEN_ADDRESS, - LINK_TOKEN_ADDRESS, - REN_TOKEN_ADDRESS, - UNI_TOKEN_ADDRESS, - USDC_TOKEN_ADDRESS, - WBTC_TOKEN_ADDRESS, - WETH_ADDRESS, - YEARN_TOKEN_ADDRESS, -] +from mev_inspect.schemas.prices import TOKEN_ADDRESSES, TOKEN_NAME_BY_ADDRESS, Price -async def fetch_all_supported_prices() -> List[Price]: +def fetch_prices() -> List[Price]: + cg = CoinGeckoAPI() prices = [] - for token_address in SUPPORTED_TOKENS: - coinbase_prices = await fetch_coinbase_prices(token_address) - for usd_price, timestamp_seconds in coinbase_prices.all.prices: - price = Price( - token_address=token_address, - usd_price=usd_price, - timestamp=timestamp_seconds, + for token_address in TOKEN_ADDRESSES: + price_data = cg.get_coin_market_chart_by_id( + id=TOKEN_NAME_BY_ADDRESS[token_address], vs_currency="usd", days="max" + ) + price_time_series = price_data["prices"] + + for entry in price_time_series: + timestamp = dt.fromtimestamp(entry[0] / 100) + token_price = entry[1] + prices.append( + Price( + timestamp=timestamp, + usd_price=token_price, + token_address=token_address, + ) ) - prices.append(price) - return prices diff --git a/mev_inspect/schemas/coinbase.py b/mev_inspect/schemas/coinbase.py deleted file mode 100644 index fca7bab..0000000 --- a/mev_inspect/schemas/coinbase.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import List, Tuple - -from pydantic import BaseModel - - -class CoinbasePricesEntry(BaseModel): - # tuple of price and timestamp - prices: List[Tuple[float, int]] - - -class CoinbasePrices(BaseModel): - all: CoinbasePricesEntry - - -class CoinbasePricesDataResponse(BaseModel): - prices: CoinbasePrices - - -class CoinbasePricesResponse(BaseModel): - data: CoinbasePricesDataResponse diff --git a/mev_inspect/schemas/prices.py b/mev_inspect/schemas/prices.py index 3c7f5fd..9227f93 100644 --- a/mev_inspect/schemas/prices.py +++ b/mev_inspect/schemas/prices.py @@ -2,6 +2,9 @@ from datetime import datetime from pydantic import BaseModel, validator +from mev_inspect.classifiers.specs.weth import WETH_ADDRESS +from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS + WBTC_TOKEN_ADDRESS = "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599" LINK_TOKEN_ADDRESS = "0x514910771af9ca656af840dff83e8264ecf986ca" YEARN_TOKEN_ADDRESS = "0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e" @@ -13,11 +16,39 @@ REN_TOKEN_ADDRESS = "0x408e41876cccdc0f92210600ef50372656052a38" CUSDC_TOKEN_ADDRESS = "0x39aa39c021dfbae8fac545936693ac917d5e7563" CDAI_TOKEN_ADDRESS = "0x5d3a536e4d6dbd6114cc1ead35777bab948e3643" +TOKEN_ADDRESSES = [ + WBTC_TOKEN_ADDRESS, + LINK_TOKEN_ADDRESS, + YEARN_TOKEN_ADDRESS, + AAVE_TOKEN_ADDRESS, + UNI_TOKEN_ADDRESS, + USDC_TOKEN_ADDRESS, + DAI_TOKEN_ADDRESS, + REN_TOKEN_ADDRESS, + CUSDC_TOKEN_ADDRESS, + CDAI_TOKEN_ADDRESS, +] + +TOKEN_NAME_BY_ADDRESS = { + WETH_ADDRESS: "weth", + ETH_TOKEN_ADDRESS: "ethereum", + WBTC_TOKEN_ADDRESS: "wrapped-bitcoin", + LINK_TOKEN_ADDRESS: "chainlink", + YEARN_TOKEN_ADDRESS: "yearn-finance", + AAVE_TOKEN_ADDRESS: "aave", + UNI_TOKEN_ADDRESS: "uniswap", + USDC_TOKEN_ADDRESS: "usd-coin", + DAI_TOKEN_ADDRESS: "dai", + REN_TOKEN_ADDRESS: "republic-protocol", + CUSDC_TOKEN_ADDRESS: "compound-usd-coin", + CDAI_TOKEN_ADDRESS: "cdai", +} + class Price(BaseModel): token_address: str + usd_price: int timestamp: datetime - usd_price: float @validator("token_address") def lower_token_address(cls, v: str) -> str: diff --git a/poetry.lock b/poetry.lock index 38c29fd..fc708b4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -730,6 +730,17 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "pycoingecko" +version = "2.2.0" +description = "Python wrapper around the CoinGecko API" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +requests = "*" + [[package]] name = "pycryptodome" version = "3.10.1" @@ -1116,7 +1127,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "2ce3bdeb2d8bd31210026e5054a54c67fc766cdf22dc83485eca425643cdf760" +content-hash = "955c3df01b275e9b4807190e468a2df4d3d18b6a45a7c1659599ef476b35be51" [metadata.files] aiohttp = [ @@ -1775,6 +1786,10 @@ py = [ {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] +pycoingecko = [ + {file = "pycoingecko-2.2.0-py3-none-any.whl", hash = "sha256:3646968c8c6936ca4e94b5f562328a763c12a0e9644141cb0215089dda59fe01"}, + {file = "pycoingecko-2.2.0.tar.gz", hash = "sha256:9add73085729b1f10f93c7948490b09e8cd47c29bebe47dccb319e8b49502d0c"}, +] pycryptodome = [ {file = "pycryptodome-3.10.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1c5e1ca507de2ad93474be5cfe2bfa76b7cf039a1a32fc196f40935944871a06"}, {file = "pycryptodome-3.10.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:6260e24d41149268122dd39d4ebd5941e9d107f49463f7e071fd397e29923b0c"}, diff --git a/pyproject.toml b/pyproject.toml index 3c4be4d..f2f9f17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ click = "^8.0.1" psycopg2 = "^2.9.1" aiohttp = "^3.8.0" dramatiq = {extras = ["redis"], version = "^1.12.1"} +pycoingecko = "^2.2.0" [tool.poetry.dev-dependencies] pre-commit = "^2.13.0" From 2a1da33752c789f69b0c75152232c2b8eba141d1 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Thu, 13 Jan 2022 10:54:00 -0500 Subject: [PATCH 2/8] Remove leftover coinbase file --- mev_inspect/coinbase.py | 46 ----------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 mev_inspect/coinbase.py diff --git a/mev_inspect/coinbase.py b/mev_inspect/coinbase.py deleted file mode 100644 index df1996c..0000000 --- a/mev_inspect/coinbase.py +++ /dev/null @@ -1,46 +0,0 @@ -import aiohttp - -from mev_inspect.classifiers.specs.weth import WETH_ADDRESS -from mev_inspect.schemas.coinbase import CoinbasePrices, CoinbasePricesResponse -from mev_inspect.schemas.prices import ( - AAVE_TOKEN_ADDRESS, - CDAI_TOKEN_ADDRESS, - CUSDC_TOKEN_ADDRESS, - DAI_TOKEN_ADDRESS, - LINK_TOKEN_ADDRESS, - REN_TOKEN_ADDRESS, - UNI_TOKEN_ADDRESS, - USDC_TOKEN_ADDRESS, - WBTC_TOKEN_ADDRESS, - YEARN_TOKEN_ADDRESS, -) -from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS - -COINBASE_API_BASE = "https://www.coinbase.com/api/v2" -COINBASE_TOKEN_NAME_BY_ADDRESS = { - WETH_ADDRESS: "weth", - ETH_TOKEN_ADDRESS: "ethereum", - WBTC_TOKEN_ADDRESS: "wrapped-bitcoin", - LINK_TOKEN_ADDRESS: "chainlink", - YEARN_TOKEN_ADDRESS: "yearn-finance", - AAVE_TOKEN_ADDRESS: "aave", - UNI_TOKEN_ADDRESS: "uniswap", - USDC_TOKEN_ADDRESS: "usdc", - DAI_TOKEN_ADDRESS: "dai", - REN_TOKEN_ADDRESS: "ren", - CUSDC_TOKEN_ADDRESS: "compound-usd-coin", - CDAI_TOKEN_ADDRESS: "compound-dai", -} - - -async def fetch_coinbase_prices(token_address: str) -> CoinbasePrices: - if token_address not in COINBASE_TOKEN_NAME_BY_ADDRESS: - raise ValueError(f"Unsupported token_address {token_address}") - - coinbase_token_name = COINBASE_TOKEN_NAME_BY_ADDRESS[token_address] - url = f"{COINBASE_API_BASE}/assets/prices/{coinbase_token_name}" - - async with aiohttp.ClientSession() as session: - async with session.get(url, params={"base": "USD"}) as response: - json_data = await response.json() - return CoinbasePricesResponse(**json_data).data.prices From 7af515d1ac54bb8035ae4692e7cf75b0528b5009 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Thu, 13 Jan 2022 11:17:48 -0500 Subject: [PATCH 3/8] Change price to float --- mev_inspect/schemas/prices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mev_inspect/schemas/prices.py b/mev_inspect/schemas/prices.py index 9227f93..3b99ad1 100644 --- a/mev_inspect/schemas/prices.py +++ b/mev_inspect/schemas/prices.py @@ -47,7 +47,7 @@ TOKEN_NAME_BY_ADDRESS = { class Price(BaseModel): token_address: str - usd_price: int + usd_price: float timestamp: datetime @validator("token_address") From 3072e4a82695a71cdffa38e94afbf213c2b14233 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Fri, 14 Jan 2022 13:17:37 -0500 Subject: [PATCH 4/8] Specify coingecko id's and remove async keyword from cli --- cli.py | 2 +- mev_inspect/prices.py | 4 ++-- mev_inspect/schemas/prices.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cli.py b/cli.py index 4cbaeb0..140268e 100644 --- a/cli.py +++ b/cli.py @@ -108,7 +108,7 @@ def enqueue_many_blocks_command(after_block: int, before_block: int, batch_size: @cli.command() @coro -async def fetch_all_prices(): +def fetch_all_prices(): inspect_db_session = get_inspect_session() logger.info("Fetching prices") diff --git a/mev_inspect/prices.py b/mev_inspect/prices.py index 94816c6..af0eff3 100644 --- a/mev_inspect/prices.py +++ b/mev_inspect/prices.py @@ -3,7 +3,7 @@ from typing import List from pycoingecko import CoinGeckoAPI -from mev_inspect.schemas.prices import TOKEN_ADDRESSES, TOKEN_NAME_BY_ADDRESS, Price +from mev_inspect.schemas.prices import COINGECKO_ID_BY_ADDRESS, TOKEN_ADDRESSES, Price def fetch_prices() -> List[Price]: @@ -12,7 +12,7 @@ def fetch_prices() -> List[Price]: for token_address in TOKEN_ADDRESSES: price_data = cg.get_coin_market_chart_by_id( - id=TOKEN_NAME_BY_ADDRESS[token_address], vs_currency="usd", days="max" + id=COINGECKO_ID_BY_ADDRESS[token_address], vs_currency="usd", days="max" ) price_time_series = price_data["prices"] diff --git a/mev_inspect/schemas/prices.py b/mev_inspect/schemas/prices.py index 3b99ad1..e57987c 100644 --- a/mev_inspect/schemas/prices.py +++ b/mev_inspect/schemas/prices.py @@ -29,7 +29,7 @@ TOKEN_ADDRESSES = [ CDAI_TOKEN_ADDRESS, ] -TOKEN_NAME_BY_ADDRESS = { +COINGECKO_ID_BY_ADDRESS = { WETH_ADDRESS: "weth", ETH_TOKEN_ADDRESS: "ethereum", WBTC_TOKEN_ADDRESS: "wrapped-bitcoin", From fed3497afca0bdde1f1f3bd3dca3dc8891b12a22 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Mon, 17 Jan 2022 22:09:27 -0500 Subject: [PATCH 5/8] Remove @coro from cli --- cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cli.py b/cli.py index 140268e..f67441d 100644 --- a/cli.py +++ b/cli.py @@ -107,7 +107,6 @@ def enqueue_many_blocks_command(after_block: int, before_block: int, batch_size: @cli.command() -@coro def fetch_all_prices(): inspect_db_session = get_inspect_session() From a3d83e625cfbd869f1b2eb1dc5dcb68e66acc6a3 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Tue, 18 Jan 2022 16:01:33 -0500 Subject: [PATCH 6/8] Add cETH and cWBTC --- mev_inspect/schemas/prices.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mev_inspect/schemas/prices.py b/mev_inspect/schemas/prices.py index e57987c..91f0961 100644 --- a/mev_inspect/schemas/prices.py +++ b/mev_inspect/schemas/prices.py @@ -15,6 +15,9 @@ DAI_TOKEN_ADDRESS = "0x6b175474e89094c44da98b954eedeac495271d0f" REN_TOKEN_ADDRESS = "0x408e41876cccdc0f92210600ef50372656052a38" CUSDC_TOKEN_ADDRESS = "0x39aa39c021dfbae8fac545936693ac917d5e7563" CDAI_TOKEN_ADDRESS = "0x5d3a536e4d6dbd6114cc1ead35777bab948e3643" +CETH_TOKEN_ADDRESS = "0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5" +CWBTC_TOKEN_ADDRESS = "0xc11b1268c1a384e55c48c2391d8d480264a3a7f4" + TOKEN_ADDRESSES = [ WBTC_TOKEN_ADDRESS, @@ -27,6 +30,8 @@ TOKEN_ADDRESSES = [ REN_TOKEN_ADDRESS, CUSDC_TOKEN_ADDRESS, CDAI_TOKEN_ADDRESS, + CETH_TOKEN_ADDRESS, + CWBTC_TOKEN_ADDRESS ] COINGECKO_ID_BY_ADDRESS = { @@ -42,6 +47,8 @@ COINGECKO_ID_BY_ADDRESS = { REN_TOKEN_ADDRESS: "republic-protocol", CUSDC_TOKEN_ADDRESS: "compound-usd-coin", CDAI_TOKEN_ADDRESS: "cdai", + CETH_TOKEN_ADDRESS: "compound-ether", + CWBTC_TOKEN_ADDRESS: "compound-wrapped-btc", } From c7de7cf808ee783cbd8ed384cef0c2c983e035d3 Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Tue, 18 Jan 2022 16:04:31 -0500 Subject: [PATCH 7/8] Fix Black pre-commit --- mev_inspect/schemas/prices.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mev_inspect/schemas/prices.py b/mev_inspect/schemas/prices.py index 91f0961..7e0b7a9 100644 --- a/mev_inspect/schemas/prices.py +++ b/mev_inspect/schemas/prices.py @@ -18,7 +18,6 @@ CDAI_TOKEN_ADDRESS = "0x5d3a536e4d6dbd6114cc1ead35777bab948e3643" CETH_TOKEN_ADDRESS = "0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5" CWBTC_TOKEN_ADDRESS = "0xc11b1268c1a384e55c48c2391d8d480264a3a7f4" - TOKEN_ADDRESSES = [ WBTC_TOKEN_ADDRESS, LINK_TOKEN_ADDRESS, @@ -31,7 +30,7 @@ TOKEN_ADDRESSES = [ CUSDC_TOKEN_ADDRESS, CDAI_TOKEN_ADDRESS, CETH_TOKEN_ADDRESS, - CWBTC_TOKEN_ADDRESS + CWBTC_TOKEN_ADDRESS, ] COINGECKO_ID_BY_ADDRESS = { From d8f896bda327093dffeba85193e53f704baa46af Mon Sep 17 00:00:00 2001 From: Gui Heise Date: Tue, 18 Jan 2022 16:33:32 -0500 Subject: [PATCH 8/8] Add Eth and Weth prices --- mev_inspect/prices.py | 5 ++++- mev_inspect/schemas/prices.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mev_inspect/prices.py b/mev_inspect/prices.py index af0eff3..e69394c 100644 --- a/mev_inspect/prices.py +++ b/mev_inspect/prices.py @@ -12,7 +12,10 @@ def fetch_prices() -> List[Price]: for token_address in TOKEN_ADDRESSES: price_data = cg.get_coin_market_chart_by_id( - id=COINGECKO_ID_BY_ADDRESS[token_address], vs_currency="usd", days="max" + id=COINGECKO_ID_BY_ADDRESS[token_address], + vs_currency="usd", + days="max", + interval="daily", ) price_time_series = price_data["prices"] diff --git a/mev_inspect/schemas/prices.py b/mev_inspect/schemas/prices.py index 7e0b7a9..b04000f 100644 --- a/mev_inspect/schemas/prices.py +++ b/mev_inspect/schemas/prices.py @@ -19,6 +19,8 @@ CETH_TOKEN_ADDRESS = "0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5" CWBTC_TOKEN_ADDRESS = "0xc11b1268c1a384e55c48c2391d8d480264a3a7f4" TOKEN_ADDRESSES = [ + ETH_TOKEN_ADDRESS, + WETH_ADDRESS, WBTC_TOKEN_ADDRESS, LINK_TOKEN_ADDRESS, YEARN_TOKEN_ADDRESS,