From 221b5aa6b4ce873caeab5f6b53e343932c5b1c77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eru=20Il=C3=BAvatar?= Date: Wed, 18 Jan 2023 19:20:51 +0100 Subject: [PATCH] feat: add chain in csv - closes #27 (#28) * feat: add chain in csv - closes #27 Signed-off-by: Arthurim * feat: add chain in csv - closes #27 Signed-off-by: Arthurim Signed-off-by: Arthurim --- profit_analysis/analysis.py | 27 +++++++++--- profit_analysis/coingecko.py | 7 +-- profit_analysis/token_utils.py | 3 +- resources/address_to_coingecko_ids.csv | 61 ++++++++++++++++---------- resources/address_to_decimals.csv | 56 +++++++++++++++-------- 5 files changed, 103 insertions(+), 51 deletions(-) diff --git a/profit_analysis/analysis.py b/profit_analysis/analysis.py index 05f5fc6..acea9a4 100644 --- a/profit_analysis/analysis.py +++ b/profit_analysis/analysis.py @@ -40,13 +40,14 @@ def analyze_profit(inspect_db_session, block_from, block_to, save_to_csv=False): profit = read_profit_from_to(inspect_db_session, block_from, block_to) w3 = create_web3() profit = add_block_timestamp(w3, profit) - profit = add_cg_ids(profit) - profit = get_usd_profit(profit, save_to_csv) + chain = get_chain_from_url(w3.provider.endpoint_uri) + profit = add_cg_ids(profit, chain) + profit = get_usd_profit(profit, chain, save_to_csv) print(profit) return profit -def get_usd_profit(profit, save_to_csv=False): +def get_usd_profit(profit, chain, save_to_csv=False): """ For each token involved in mev transactions, will get its price at the time of the transaction and compute the profit of each mev transaction. @@ -54,6 +55,7 @@ def get_usd_profit(profit, save_to_csv=False): :param profit: pd.DataFrame, with columns = ['block_number', 'timestamp', 'transaction_hash', 'token_debt', 'amount_debt', 'cg_id_debt', 'token_received', 'amount_received', 'cg_id_received'] + :param chain: str, the blockchain :param save_to_csv: bool, whether to save the analysed profits to csv or not :return: pd.DataFrame, with columns = ['block_number', 'timestamp', 'date', 'transaction_hash', 'amount_received', 'token_received', 'price_received', @@ -61,7 +63,7 @@ def get_usd_profit(profit, save_to_csv=False): 'profit_usd' ] """ tokens = profit[CG_ID_RECEIVED_KEY].unique() - mapping = get_address_to_coingecko_ids_mapping() + mapping = get_address_to_coingecko_ids_mapping(chain) profit_with_price_tokens = pd.DataFrame() failures = {} for token in tokens: @@ -93,7 +95,7 @@ def get_usd_profit(profit, save_to_csv=False): # get received token decimals decimals = get_decimals( - profit_by_received_token[TOKEN_RECEIVED_KEY].values[0] + profit_by_received_token[TOKEN_RECEIVED_KEY].values[0], chain ) # get debt tokens prices @@ -125,7 +127,7 @@ def get_usd_profit(profit, save_to_csv=False): profit_by_received_token[TOKEN_DEBT_KEY].astype(str).unique().tolist() ): if debt_token != "": - debt_token_decimals = get_decimals(debt_token) + debt_token_decimals = get_decimals(debt_token, chain) debt_tokens_decimals = pd.concat( [ debt_tokens_decimals, @@ -260,3 +262,16 @@ def create_web3(): return w3_provider else: raise Exception("Failed to connect") + + +def get_chain_from_url(url): + if "ether" in url: + return "ethereum" + elif "poly" in url: + return "polygon" + elif "arb" in url: + return "arbitrum" + elif "opti" in url: + return "optimism" + else: + raise Exception(f"Could not determine blockchain from url: {url}") diff --git a/profit_analysis/coingecko.py b/profit_analysis/coingecko.py index f6c3cb6..4448430 100644 --- a/profit_analysis/coingecko.py +++ b/profit_analysis/coingecko.py @@ -15,8 +15,9 @@ from profit_analysis.constants import DATA_PATH TRAILING_ZEROS = "000000000000000000000000" -def get_address_to_coingecko_ids_mapping(): +def get_address_to_coingecko_ids_mapping(chain): token_cg_ids = pd.read_csv(DATA_PATH + "address_to_coingecko_ids.csv") + token_cg_ids = token_cg_ids.loc[token_cg_ids["chain"] == chain] token_cg_ids[TOKEN_DEBT_KEY] = token_cg_ids[TOKEN_KEY].astype(str) token_cg_ids[CG_ID_RECEIVED_KEY] = token_cg_ids[CG_ID_KEY] token_cg_ids[CG_ID_DEBT_KEY] = token_cg_ids[CG_ID_KEY] @@ -24,8 +25,8 @@ def get_address_to_coingecko_ids_mapping(): return token_cg_ids -def add_cg_ids(profit_by_block): - token_cg_ids = get_address_to_coingecko_ids_mapping() +def add_cg_ids(profit_by_block, chain): + token_cg_ids = get_address_to_coingecko_ids_mapping(chain) token_cg_ids[TOKEN_DEBT_KEY] = token_cg_ids[TOKEN_DEBT_KEY].str.lower() token_cg_ids[TOKEN_RECEIVED_KEY] = token_cg_ids[TOKEN_RECEIVED_KEY].str.lower() profit_by_block[TOKEN_RECEIVED_KEY] = ( diff --git a/profit_analysis/token_utils.py b/profit_analysis/token_utils.py index d9661f0..867628c 100644 --- a/profit_analysis/token_utils.py +++ b/profit_analysis/token_utils.py @@ -3,8 +3,9 @@ from profit_analysis.column_names import TOKEN_KEY from profit_analysis.constants import DATA_PATH -def get_decimals(token_address): +def get_decimals(token_address, chain): decimals_mapping = pd.read_csv(DATA_PATH + "address_to_decimals.csv") + decimals_mapping = decimals_mapping.loc[decimals_mapping["chain"] == chain] decimals_mapping[TOKEN_KEY] = decimals_mapping[TOKEN_KEY].str.lower() decimals = decimals_mapping.loc[ decimals_mapping[TOKEN_KEY] == token_address.lower(), "decimals" diff --git a/resources/address_to_coingecko_ids.csv b/resources/address_to_coingecko_ids.csv index cfcfd66..59e33e0 100644 --- a/resources/address_to_coingecko_ids.csv +++ b/resources/address_to_coingecko_ids.csv @@ -1,23 +1,38 @@ -token,cg_id -0x7ceB23fD6bC0adD59E62ac25578270cFf1b9f619,weth -0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee,ethereum -0x2260fac5e5542a773aa44fbcfedf7c193bc2c599,wrapped-bitcoin -0x514910771af9ca656af840dff83e8264ecf986ca,chainlink -0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e,yearn-finance -0xD6DF932A45C0f255f85145f286eA0b292B21C90B,aave -0x1f9840a85d5af5bf1d1762f925bdaddc4201f984,uniswap -0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174,usd-coin -0x6b175474e89094c44da98b954eedeac495271d0f,dai -0x408e41876cccdc0f92210600ef50372656052a38,republic-protocol -0x39aa39c021dfbae8fac545936693ac917d5e7563,compound-usd-coin -0x5d3a536e4d6dbd6114cc1ead35777bab948e3643,cdai -0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5,compound-ether -0xc11b1268c1a384e55c48c2391d8d480264a3a7f4,compound-wrapped-btc -0x0d500B1d8E8eF31E21C99d1Db9A6444d3ADf1270,matic-network -0x430EF9263E76DAE63c84292C3409D61c598E9682,vulcan-forged -0xc2132D05D31c914a87C6611C10748AEb04B58e8F,tether -0x22a31bD4cB694433B6de19e0aCC2899E553e9481,mmfinance -0xE6469Ba6D2fD6130788E0eA9C0a0515900563b59,terrausd-wormhole -0x6f8a06447Ff6FcF75d803135a7de15CE88C1d4ec,shiba-inu -0x831753DD7087CaC61aB5644b308642cc1c33Dc13,quick -0x723b17718289a91af252d616de2c77944962d122,gaia-everworld \ No newline at end of file +chain,token,cg_id +ethereum,0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee,ethereum +ethereum,0x2260fac5e5542a773aa44fbcfedf7c193bc2c599,wrapped-bitcoin +ethereum,0x514910771af9ca656af840dff83e8264ecf986ca,chainlink +ethereum,0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e,yearn-finance +ethereum,0x6b175474e89094c44da98b954eedeac495271d0f,dai +ethereum,0xdAC17F958D2ee523a2206206994597C13D831ec7,tether +ethereum,0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48,usd-coin +ethereum,0xB8c77482e45F1F44dE1745F52C74426C631bDD52,binancecoin +ethereum,0x4Fabb145d64652a948d72533023f6E7A623C7C53,binance-usd +ethereum,0x7D1AfA7B718fb893dB30A3aBc0Cfc608AaCfeBB0,matic-network +ethereum,0xae7ab96520DE3A18E5e111B5EaAb095312D7fE84,staked-ether +ethereum,0x408e41876cccdc0f92210600ef50372656052a38,republic-protocol +ethereum,0x39aa39c021dfbae8fac545936693ac917d5e7563,compound-usd-coin +ethereum,0x5d3a536e4d6dbd6114cc1ead35777bab948e3643,cdai +ethereum,0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5,compound-ether +ethereum,0xc11b1268c1a384e55c48c2391d8d480264a3a7f4,compound-wrapped-btc +ethereum,0x430EF9263E76DAE63c84292C3409D61c598E9682,vulcan-forged +ethereum,0x1f9840a85d5af5bf1d1762f925bdaddc4201f984,uniswap +polygon,0x7ceB23fD6bC0adD59E62ac25578270cFf1b9f619,weth +polygon,0xD6DF932A45C0f255f85145f286eA0b292B21C90B,aave +polygon,0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174,usd-coin +polygon,0x0d500B1d8E8eF31E21C99d1Db9A6444d3ADf1270,matic-network +polygon,0x430EF9263E76DAE63c84292C3409D61c598E9682,vulcan-forged +polygon,0xc2132D05D31c914a87C6611C10748AEb04B58e8F,tether +polygon,0x3BA4c387f786bFEE076A58914F5Bd38d668B42c3,binancecoin +polygon,0xdAb529f40E671A1D4bF91361c21bf9f0C9712ab7,binance-usd +polygon,0x8f3Cf7ad23Cd3CaDbD9735AFf958023239c6A063,dai +polygon,0x2C89bbc92BD86F8075d1DEcc58C7F4E0107f286b,avalanche-2 +polygon,0xb33EaAd8d922B1083446DC23f610c2567fB5180f,uniswap +polygon,0x1BFD67037B42Cf73acF2047067bd4F2C47D9BfD6,wrapped-bitcoin +polygon,0xb0897686c545045aFc77CF20eC7A532E3120E0F1,chainlink +polygon,0x53E0bca35eC356BD5ddDFebbD1Fc0fD03FaBad39,chainlink +polygon,0x22a31bD4cB694433B6de19e0aCC2899E553e9481,mmfinance +polygon,0xE6469Ba6D2fD6130788E0eA9C0a0515900563b59,terrausd-wormhole +polygon,0x6f8a06447Ff6FcF75d803135a7de15CE88C1d4ec,shiba-inu +polygon,0x831753DD7087CaC61aB5644b308642cc1c33Dc13,quick +polygon,0x723b17718289a91af252d616de2c77944962d122,gaia-everworld \ No newline at end of file diff --git a/resources/address_to_decimals.csv b/resources/address_to_decimals.csv index d3ccf53..4769c53 100644 --- a/resources/address_to_decimals.csv +++ b/resources/address_to_decimals.csv @@ -1,18 +1,38 @@ -token,decimals -0x7ceB23fD6bC0adD59E62ac25578270cFf1b9f619,18 -0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee,18 -0x2260fac5e5542a773aa44fbcfedf7c193bc2c599,18 -0x514910771af9ca656af840dff83e8264ecf986ca,18 -0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e,18 -0xD6DF932A45C0f255f85145f286eA0b292B21C90B,18 -0x1f9840a85d5af5bf1d1762f925bdaddc4201f984,18 -0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174,6 -0x39aa39c021dfbae8fac545936693ac917d5e7563,8 -0x0d500B1d8E8eF31E21C99d1Db9A6444d3ADf1270,18 -0x430EF9263E76DAE63c84292C3409D61c598E9682,18 -0xc2132D05D31c914a87C6611C10748AEb04B58e8F,6 -0x22a31bD4cB694433B6de19e0aCC2899E553e9481,18 -0xE6469Ba6D2fD6130788E0eA9C0a0515900563b59,6 -0x6f8a06447Ff6FcF75d803135a7de15CE88C1d4ec,18 -0x831753DD7087CaC61aB5644b308642cc1c33Dc13,18 -0x723b17718289a91af252d616de2c77944962d122,18 \ No newline at end of file +chain,token,decimals +ethereum,0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee,18 +ethereum,0x2260fac5e5542a773aa44fbcfedf7c193bc2c599,18 +ethereum,0x514910771af9ca656af840dff83e8264ecf986ca,18 +ethereum,0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e,18 +ethereum,0x6b175474e89094c44da98b954eedeac495271d0f,18 +ethereum,0xdAC17F958D2ee523a2206206994597C13D831ec7,6 +ethereum,0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48,6 +ethereum,0xB8c77482e45F1F44dE1745F52C74426C631bDD52,18 +ethereum,0x4Fabb145d64652a948d72533023f6E7A623C7C53,18 +ethereum,0x7D1AfA7B718fb893dB30A3aBc0Cfc608AaCfeBB0,18 +ethereum,0xae7ab96520DE3A18E5e111B5EaAb095312D7fE84,18 +ethereum,0x408e41876cccdc0f92210600ef50372656052a38,18 +ethereum,0x39aa39c021dfbae8fac545936693ac917d5e7563,8 +ethereum,0x5d3a536e4d6dbd6114cc1ead35777bab948e3643,8 +ethereum,0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5,8 +ethereum,0xc11b1268c1a384e55c48c2391d8d480264a3a7f4,8 +ethereum,0x430EF9263E76DAE63c84292C3409D61c598E9682,18 +ethereum,0x1f9840a85d5af5bf1d1762f925bdaddc4201f984,18 +polygon,0x7ceB23fD6bC0adD59E62ac25578270cFf1b9f619,18 +polygon,0xD6DF932A45C0f255f85145f286eA0b292B21C90B,18 +polygon,0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174,6 +polygon,0x0d500B1d8E8eF31E21C99d1Db9A6444d3ADf1270,18 +polygon,0x430EF9263E76DAE63c84292C3409D61c598E9682,18 +polygon,0xc2132D05D31c914a87C6611C10748AEb04B58e8F,6 +polygon,0x3BA4c387f786bFEE076A58914F5Bd38d668B42c3,18 +polygon,0xdAb529f40E671A1D4bF91361c21bf9f0C9712ab7,18 +polygon,0x8f3Cf7ad23Cd3CaDbD9735AFf958023239c6A063,18 +polygon,0x2C89bbc92BD86F8075d1DEcc58C7F4E0107f286b,18 +polygon,0xb33EaAd8d922B1083446DC23f610c2567fB5180f,18 +polygon,0x1BFD67037B42Cf73acF2047067bd4F2C47D9BfD6,8 +polygon,0xb0897686c545045aFc77CF20eC7A532E3120E0F1,18 +polygon,0x53E0bca35eC356BD5ddDFebbD1Fc0fD03FaBad39,18 +polygon,0x22a31bD4cB694433B6de19e0aCC2899E553e9481,18 +polygon,0xE6469Ba6D2fD6130788E0eA9C0a0515900563b59,6 +polygon,0x6f8a06447Ff6FcF75d803135a7de15CE88C1d4ec,18 +polygon,0x831753DD7087CaC61aB5644b308642cc1c33Dc13,18 +polygon,0x723b17718289a91af252d616de2c77944962d122,18 \ No newline at end of file