diff --git a/packages/pipeline/migrations/1550749543417-CreateEtherscanTransactions.ts b/packages/pipeline/migrations/1550749543417-CreateEtherscanTransactions.ts new file mode 100644 index 0000000000..4b0dbd7075 --- /dev/null +++ b/packages/pipeline/migrations/1550749543417-CreateEtherscanTransactions.ts @@ -0,0 +1,36 @@ +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; + +const table = new Table({ + name: 'raw.etherscan_transactions', + columns: [ + { name: 'hash', type: 'varchar', isPrimary: true }, + + { name: 'block_number', type: 'numeric', isNullable: false }, + { name: 'timestamp', type: 'numeric', isNullable: false }, + { name: 'block_hash', type: 'varchar', isNullable: false }, + { name: 'transaction_index', type: 'numeric', isNullable: false }, + { name: 'nonce', type: 'numeric', isNullable: false }, + { name: 'from', type: 'varchar', isNullable: false }, + { name: 'to', type: 'varchar', isNullable: false }, + { name: 'value', type: 'numeric', isNullable: false }, + { name: 'gas', type: 'numeric', isNullable: false }, + { name: 'gas_price', type: 'numeric', isNullable: false }, + { name: 'is_error', type: 'boolean', isNullable: false }, + { name: 'txreceipt_status', type: 'varchar', isNullable: true }, + { name: 'input', type: 'varchar', isNullable: false }, + { name: 'contract_address', type: 'varchar', isNullable: false }, + { name: 'cumulative_gas_used', type: 'numeric', isNullable: false }, + { name: 'gas_used', type: 'numeric', isNullable: false }, + { name: 'confirmations', type: 'numeric', isNullable: false }, + ], +}); + +export class CreateEtherscanTransactions1550749543417 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable(table); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable(table); + } +} diff --git a/packages/pipeline/src/data_sources/etherscan/index.ts b/packages/pipeline/src/data_sources/etherscan/index.ts new file mode 100644 index 0000000000..8a0746768a --- /dev/null +++ b/packages/pipeline/src/data_sources/etherscan/index.ts @@ -0,0 +1,50 @@ +import { fetchAsync } from '@0x/utils'; + +const TIMEOUT = 240000; + +export interface EtherscanResponse { + status: string; + message: string; + result: EtherscanTransactionResponse[]; +} + +export interface EtherscanTransactionResponse { + blockNumber: string; + timeStamp: string; + hash: string; + nonce: string; + blockHash: string; + transactionIndex: string; + from: string; + to: string; + value: string; + gas: string; + gasPrice: string; + isError: string; + txreceipt_status: string; + input: string; + contractAddress: string; + cumulativeGasUsed: string; + gasUsed: string; + confirmations: string; +} + +// tslint:disable:prefer-function-over-method +// ^ Keep consistency with other sources and help logical organization +export class EtherscanSource { + public readonly _urlBase: string; + + constructor(apiKey: string, startBlock: number, endBlock: number) { + this._urlBase = `http://api.etherscan.io/api?module=account&action=txlist&startblock=${startBlock}&endblock=${endBlock}&sort=asc&apikey=${apiKey}&address=`; + } + + /** + * Call Etherscan API and return result. + */ + public async getEtherscanTransactionsForAddressAsync(address: string): Promise { + const urlWithAddress = this._urlBase + address; + const resp = await fetchAsync(urlWithAddress, {}, TIMEOUT); + const respJson: EtherscanResponse = await resp.json(); + return respJson.result; + } +} diff --git a/packages/pipeline/src/entities/etherscan_transaction.ts b/packages/pipeline/src/entities/etherscan_transaction.ts new file mode 100644 index 0000000000..c84eddc978 --- /dev/null +++ b/packages/pipeline/src/entities/etherscan_transaction.ts @@ -0,0 +1,61 @@ +import { BigNumber } from '@0x/utils'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'etherscan_transactions', schema: 'raw' }) +export class EtherscanTransaction { + @PrimaryColumn({ name: 'hash' }) + public hash!: string; + + @Column({ name: 'block_number', type: 'bigint', transformer: bigNumberTransformer }) + public blockNumber!: BigNumber; + + @Column({ name: 'timestamp', type: 'bigint', transformer: bigNumberTransformer }) + public timeStamp!: BigNumber; + + @Column({ name: 'block_hash' }) + public blockHash!: string; + + @Column({ name: 'transaction_index', transformer: numberToBigIntTransformer }) + public transactionIndex!: number; + + @Column({ name: 'nonce', transformer: numberToBigIntTransformer }) + public nonce!: number; + + @Column({ name: 'from' }) + public from!: string; + + @Column({ name: 'to' }) + public to!: string; + + @Column({ name: 'value', type: 'bigint', transformer: bigNumberTransformer }) + public value!: BigNumber; + + @Column({ name: 'gas', type: 'bigint', transformer: bigNumberTransformer }) + public gas!: BigNumber; + + @Column({ name: 'gas_price', type: 'bigint', transformer: bigNumberTransformer }) + public gasPrice!: BigNumber; + + @Column({ name: 'is_error', type: 'boolean', nullable: true }) + public isError!: boolean; + + @Column({ name: 'txreceipt_status' }) + public txreceiptStatus?: string; + + @Column({ name: 'input' }) + public input!: string; + + @Column({ name: 'contract_address' }) + public contractAddress!: string; + + @Column({ name: 'cumulative_gas_used', type: 'bigint', transformer: bigNumberTransformer }) + public cumulativeGasUsed!: BigNumber; + + @Column({ name: 'gas_used', type: 'bigint', transformer: bigNumberTransformer }) + public gasUsed!: BigNumber; + + @Column({ name: 'confirmations', type: 'bigint', transformer: bigNumberTransformer }) + public confirmations!: BigNumber; +} diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts index 174a32a446..0f3e85f9c6 100644 --- a/packages/pipeline/src/entities/index.ts +++ b/packages/pipeline/src/entities/index.ts @@ -4,6 +4,7 @@ import { ExchangeFillEvent } from './exchange_fill_event'; export { Block } from './block'; export { DexTrade } from './dex_trade'; +export { EtherscanTransaction } from './etherscan_transaction'; export { ExchangeCancelEvent } from './exchange_cancel_event'; export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event'; export { ExchangeFillEvent } from './exchange_fill_event'; diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 4604686acf..e2e6be7691 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -9,6 +9,7 @@ import { CopperOpportunity, DexTrade, ERC20ApprovalEvent, + EtherscanTransaction, ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, @@ -31,6 +32,7 @@ const entities = [ CopperCustomField, CopperLead, DexTrade, + EtherscanTransaction, ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, diff --git a/packages/pipeline/src/parsers/etherscan/index.ts b/packages/pipeline/src/parsers/etherscan/index.ts new file mode 100644 index 0000000000..24ecf0445c --- /dev/null +++ b/packages/pipeline/src/parsers/etherscan/index.ts @@ -0,0 +1,42 @@ +import { BigNumber } from '@0x/utils'; +import * as R from 'ramda'; + +import { EtherscanTransactionResponse } from '../../data_sources/etherscan'; +import { EtherscanTransaction } from '../../entities'; + +/** + * Parses an Etherscan response from the Etherscan API and returns an array of + * EtherscanTransaction entities. + * @param rawTrades A raw order response from an SRA endpoint. + */ +export function parseEtherscanTransactions(rawTransactions: EtherscanTransactionResponse[]): EtherscanTransaction[] { + return R.map(_parseEtherscanTransaction, rawTransactions); +} + +/** + * Converts a single Etherscan transction into an EtherscanTransaction entity. + * @param rawTx A single Etherscan transaction from the Etherscan API. + */ +export function _parseEtherscanTransaction(rawTx: EtherscanTransactionResponse): EtherscanTransaction { + const parsedTx = new EtherscanTransaction(); + parsedTx.blockNumber = new BigNumber(rawTx.blockNumber); + parsedTx.timeStamp = new BigNumber(rawTx.timeStamp); + parsedTx.hash = rawTx.hash; + parsedTx.blockHash = rawTx.blockHash; + parsedTx.transactionIndex = Number(rawTx.transactionIndex); + parsedTx.nonce = Number(rawTx.nonce); + parsedTx.from = rawTx.from; + parsedTx.to = rawTx.to; + parsedTx.value = new BigNumber(rawTx.value); + parsedTx.gas = new BigNumber(rawTx.gas); + parsedTx.gasPrice = new BigNumber(rawTx.gasPrice); + parsedTx.isError = rawTx.isError === '0' ? false : true; + parsedTx.txreceiptStatus = rawTx.txreceipt_status; + parsedTx.input = rawTx.input; + parsedTx.contractAddress = rawTx.contractAddress; + parsedTx.cumulativeGasUsed = new BigNumber(rawTx.cumulativeGasUsed); + parsedTx.gasUsed = new BigNumber(rawTx.gasUsed); + parsedTx.confirmations = new BigNumber(rawTx.confirmations); + + return parsedTx; +} diff --git a/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts b/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts new file mode 100644 index 0000000000..23c8cf4079 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts @@ -0,0 +1,66 @@ +import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm'; + +import { getContractAddressesForNetworkOrThrow, NetworkId } from '@0x/contract-addresses'; +import { web3Factory } from '@0x/dev-utils'; +import { Web3ProviderEngine } from '@0x/subproviders'; +import { logUtils } from '@0x/utils'; +import { Web3Wrapper } from '@0x/web3-wrapper'; + +import { EtherscanSource } from '../data_sources/etherscan'; +import { EtherscanTransaction } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseEtherscanTransactions } from '../parsers/etherscan'; +import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils'; + +const BATCH_SAVE_SIZE = 1000; // Number of orders to save at once. +const START_BLOCK_OFFSET = 100; // Number of blocks before the last known block to consider when updating fill events. +const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default endBlock. + +let connection: Connection; + +(async () => { + const apiKey = process.env.ETHERSCAN_API_KEY; + if (apiKey === undefined) { + throw new Error('Missing required env var: ETHERSCAN_API_KEY'); + } + connection = await createConnection(ormConfig as ConnectionOptions); + const provider = web3Factory.getRpcProvider({ + rpcUrl: INFURA_ROOT_URL, + }); + const EtherscanTransactionRepository = connection.getRepository(EtherscanTransaction); + const startBlock = await getStartBlockAsync(EtherscanTransactionRepository); + logUtils.log(`Start block: ${startBlock}`); + const endBlock = await calculateEndBlockAsync(provider); + logUtils.log(`End block: ${endBlock}`); + const etherscanSource = new EtherscanSource(apiKey, startBlock, endBlock); + const exchangeContractAddress = getContractAddressesForNetworkOrThrow(NetworkId.Mainnet).exchange; + logUtils.log('Fetching exchange transactions from Etherscan...'); + const rawTransactions = await etherscanSource.getEtherscanTransactionsForAddressAsync(exchangeContractAddress); + const transactions = parseEtherscanTransactions(rawTransactions); + logUtils.log(`Saving ${transactions.length} records to database`); + await EtherscanTransactionRepository.save(transactions, { + chunk: Math.ceil(transactions.length / BATCH_SAVE_SIZE), + }); + logUtils.log('Done'); + process.exit(0); +})().catch(handleError); + +async function getStartBlockAsync(repository: Repository): Promise { + const transactionsCount = await repository.count(); + if (transactionsCount === 0) { + logUtils.log(`No existing ${repository.metadata.name}s found.`); + return EXCHANGE_START_BLOCK; + } + const tableName = repository.metadata.tableName; + const queryResult = await connection.query( + `SELECT block_number FROM raw.${tableName} ORDER BY block_number DESC LIMIT 1`, + ); + const lastKnownBlock = queryResult[0].block_number; + return lastKnownBlock - START_BLOCK_OFFSET; +} + +async function calculateEndBlockAsync(provider: Web3ProviderEngine): Promise { + const web3Wrapper = new Web3Wrapper(provider); + const currentBlock = await web3Wrapper.getBlockNumberAsync(); + return currentBlock - BLOCK_FINALITY_THRESHOLD; +} diff --git a/packages/pipeline/test/entities/etherscan_test.ts b/packages/pipeline/test/entities/etherscan_test.ts new file mode 100644 index 0000000000..7b35d92b78 --- /dev/null +++ b/packages/pipeline/test/entities/etherscan_test.ts @@ -0,0 +1,44 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; +import 'reflect-metadata'; + +import { EtherscanTransaction } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const transaction: EtherscanTransaction = { + blockNumber: new BigNumber('6271590'), + timeStamp: new BigNumber('1536083185'), + hash: '0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea', + nonce: 2, + blockHash: '0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee', + transactionIndex: 3, + from: '0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713', + to: '', + value: new BigNumber('0'), + gas: new BigNumber('7000000'), + gasPrice: new BigNumber('20000000000'), + isError: false, + txreceiptStatus: '1', + input: '0x60806040', // shortened + contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + cumulativeGasUsed: new BigNumber('6068925'), + gasUsed: new BigNumber('6005925'), + confirmations: new BigNumber('976529'), +}; + +// tslint:disable:custom-no-magic-numbers +describe('EtherscanTransaction entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const transactions = [transaction]; + const tradesRepository = connection.getRepository(EtherscanTransaction); + for (const tx of transactions) { + await testSaveAndFindEntityAsync(tradesRepository, tx); + } + }); +}); diff --git a/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.json b/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.json new file mode 100644 index 0000000000..fb2fe1b95d --- /dev/null +++ b/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.json @@ -0,0 +1,26 @@ +{ + "message": "OK", + "result": [ + { + "blockHash": "0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee", + "blockNumber": "6271590", + "confirmations": "996941", + "contractAddress": "0x4f833a24e1f95d70f028921e27040ca56e09ab0b", + "cumulativeGasUsed": "6068925", + "from": "0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713", + "gas": "7000000", + "gasPrice": "20000000000", + "gasUsed": "6005925", + "hash": "0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea", + "input": "0x60806040", + "isError": "0", + "nonce": "2", + "timeStamp": "1536083185", + "to": "", + "transactionIndex": "3", + "txreceipt_status": "1", + "value": "0" + } + ], + "status": "1" +} diff --git a/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.ts b/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.ts new file mode 100644 index 0000000000..95a81d8833 --- /dev/null +++ b/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.ts @@ -0,0 +1,30 @@ +import { BigNumber } from '@0x/utils'; + +import { EtherscanTransaction } from '../../../src/entities'; + +// To re-create the JSON file from the API (e.g. if the API output schema changes), run the below command: +// curl "https://api.etherscan.io/api?module=account&action=txlist&address=0x4f833a24e1f95d70f028921e27040ca56e09ab0b&startblock=0&endblock=99999999&page=1&offset=1&sort=asc&apikey=YourApiKeyToken" | python -m json.tool > api_v1_accounts_transactions.json + +const ParsedEtherscanTransactions: EtherscanTransaction[] = [ + { + blockNumber: new BigNumber('6271590'), + timeStamp: new BigNumber('1536083185'), + hash: '0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea', + nonce: 2, + blockHash: '0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee', + transactionIndex: 3, + from: '0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713', + to: '', + value: new BigNumber('0'), + gas: new BigNumber('7000000'), + gasPrice: new BigNumber('20000000000'), + isError: false, + txreceiptStatus: '1', + input: '0x60806040', // shortened + contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + cumulativeGasUsed: new BigNumber('6068925'), + gasUsed: new BigNumber('6005925'), + confirmations: new BigNumber('996941'), + }, +]; +export { ParsedEtherscanTransactions }; diff --git a/packages/pipeline/test/parsers/etherscan/index_test.ts b/packages/pipeline/test/parsers/etherscan/index_test.ts new file mode 100644 index 0000000000..8c021299a1 --- /dev/null +++ b/packages/pipeline/test/parsers/etherscan/index_test.ts @@ -0,0 +1,24 @@ +import * as chai from 'chai'; +import 'mocha'; + +import { EtherscanResponse } from '../../../src/data_sources/etherscan'; +import { parseEtherscanTransactions } from '../../../src/parsers/etherscan'; +import { chaiSetup } from '../../utils/chai_setup'; + +import { ParsedEtherscanTransactions } from '../../fixtures/etherscan/api_v1_accounts_transactions'; +import * as etherscanResponse from '../../fixtures/etherscan/api_v1_accounts_transactions.json'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('etherscan_transactions', () => { + describe('parseEtherscanTransactions', () => { + it('converts etherscanTransactions to EtherscanTransaction entities', () => { + const response: EtherscanResponse = etherscanResponse; + const expected = ParsedEtherscanTransactions; + const actual = parseEtherscanTransactions(response.result); + expect(actual).deep.equal(expected); + }); + }); +}); diff --git a/packages/pipeline/tsconfig.json b/packages/pipeline/tsconfig.json index 45e07374c4..4c43775d74 100644 --- a/packages/pipeline/tsconfig.json +++ b/packages/pipeline/tsconfig.json @@ -13,6 +13,7 @@ "./test/fixtures/copper/api_v1_custom_field_definitions.json", "./test/fixtures/copper/api_v1_list_activities.json", "./test/fixtures/copper/api_v1_list_leads.json", - "./test/fixtures/copper/api_v1_list_opportunities.json" + "./test/fixtures/copper/api_v1_list_opportunities.json", + "./test/fixtures/etherscan/api_v1_accounts_transactions.json" ] }