From 8885f543aeddee3b27e77329326fcfb27ed24025 Mon Sep 17 00:00:00 2001 From: askeluv Date: Thu, 21 Feb 2019 20:37:55 +0800 Subject: [PATCH 1/6] Pull exchange contract transactions from Etherscan --- ...50749543417-CreateEtherscanTransactions.ts | 38 +++++++++++ .../src/data_sources/etherscan/index.ts | 51 ++++++++++++++ .../src/entities/etherscan_transaction.ts | 62 +++++++++++++++++ packages/pipeline/src/entities/index.ts | 1 + packages/pipeline/src/ormconfig.ts | 2 + .../pipeline/src/parsers/etherscan/index.ts | 42 ++++++++++++ ...ll_exchange_transactions_from_etherscan.ts | 68 +++++++++++++++++++ .../pipeline/test/entities/etherscan_test.ts | 44 ++++++++++++ .../test/parsers/etherscan/index_test.ts | 63 +++++++++++++++++ 9 files changed, 371 insertions(+) create mode 100644 packages/pipeline/migrations/1550749543417-CreateEtherscanTransactions.ts create mode 100644 packages/pipeline/src/data_sources/etherscan/index.ts create mode 100644 packages/pipeline/src/entities/etherscan_transaction.ts create mode 100644 packages/pipeline/src/parsers/etherscan/index.ts create mode 100644 packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts create mode 100644 packages/pipeline/test/entities/etherscan_test.ts create mode 100644 packages/pipeline/test/parsers/etherscan/index_test.ts diff --git a/packages/pipeline/migrations/1550749543417-CreateEtherscanTransactions.ts b/packages/pipeline/migrations/1550749543417-CreateEtherscanTransactions.ts new file mode 100644 index 0000000000..b35f3a6e45 --- /dev/null +++ b/packages/pipeline/migrations/1550749543417-CreateEtherscanTransactions.ts @@ -0,0 +1,38 @@ +import { MigrationInterface, QueryRunner, Table } from "typeorm"; + +const table = new Table({ + name: 'raw.etherscan_transactions', + columns: [ + { name: 'hash', type: 'varchar', isPrimary: true }, + + { name: 'block_number', type: 'numeric', isNullable: false }, + { name: 'timestamp', type: 'numeric', isNullable: false }, + { name: 'block_hash', type: 'varchar', isNullable: false }, + { name: 'transaction_index', type: 'numeric', isNullable: false }, + { name: 'nonce', type: 'numeric', isNullable: false }, + { name: 'from', type: 'varchar', isNullable: false }, + { name: 'to', type: 'varchar', isNullable: false }, + { name: 'value', type: 'numeric', isNullable: false }, + { name: 'gas', type: 'numeric', isNullable: false }, + { name: 'gas_price', type: 'numeric', isNullable: false }, + { name: 'is_error', type: 'boolean', isNullable: false }, + { name: 'txreceipt_status', type: 'varchar', isNullable: true }, + { name: 'input', type: 'varchar', isNullable: false }, + { name: 'contract_address', type: 'varchar', isNullable: false }, + { name: 'cumulative_gas_used', type: 'numeric', isNullable: false }, + { name: 'gas_used', type: 'numeric', isNullable: false }, + { name: 'confirmations', type: 'numeric', isNullable: false }, + + ], +}); + +export class CreateEtherscanTransactions1550749543417 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable(table); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable(table); + } +} diff --git a/packages/pipeline/src/data_sources/etherscan/index.ts b/packages/pipeline/src/data_sources/etherscan/index.ts new file mode 100644 index 0000000000..96a5266a92 --- /dev/null +++ b/packages/pipeline/src/data_sources/etherscan/index.ts @@ -0,0 +1,51 @@ +import { fetchAsync } from '@0x/utils'; + +const TIMEOUT = 120000; + +export interface EtherscanResponse { + status: string; + message: string; + result: EtherscanTransactionResponse[]; +} + +export interface EtherscanTransactionResponse { + blockNumber: string; + timeStamp: string; + hash: string; + nonce: string; + blockHash: string; + transactionIndex: string; + from: string; + to: string; + value: string; + gas: string; + gasPrice: string; + isError: string; + txreceiptStatus: string; + input: string; + contractAddress: string; + cumulativeGasUsed: string; + gasUsed: string; + confirmations: string; +} + +// tslint:disable:prefer-function-over-method +// ^ Keep consistency with other sources and help logical organization +export class EtherscanSource { + + public readonly _urlBase: string; + + constructor(apiKey: string, startBlock: number, endBlock: number) { + this._urlBase = `http://api.etherscan.io/api?module=account&action=txlist&startblock=${startBlock}&endblock=${endBlock}&sort=desc&apikey=${apiKey}&address=`; + } + + /** + * Call Etherscan API and return result. + */ + public async getEtherscanTransactionsForAddressAsync(address: string): Promise { + const urlWithAddress = this._urlBase + address; + const resp = await fetchAsync(urlWithAddress, {}, TIMEOUT); + const respJson: EtherscanResponse = await resp.json(); + return respJson.result; + } +} diff --git a/packages/pipeline/src/entities/etherscan_transaction.ts b/packages/pipeline/src/entities/etherscan_transaction.ts new file mode 100644 index 0000000000..3de6f4977d --- /dev/null +++ b/packages/pipeline/src/entities/etherscan_transaction.ts @@ -0,0 +1,62 @@ +import { BigNumber } from '@0x/utils'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'etherscan_transactions', schema: 'raw' }) +export class EtherscanTransaction { + @PrimaryColumn({ name: 'hash' }) + public hash!: string; + + @Column({ name: 'block_number', type: 'bigint', transformer: bigNumberTransformer }) + public blockNumber!: BigNumber; + + @Column({ name: 'timestamp', type: 'bigint', transformer: bigNumberTransformer }) + public timeStamp!: BigNumber; + + @Column({ name: 'block_hash' }) + public blockHash!: string; + + @Column({ name: 'transaction_index', transformer: numberToBigIntTransformer }) + public transactionIndex!: number; + + @Column({ name: 'nonce', transformer: numberToBigIntTransformer }) + public nonce!: number; + + @Column({ name: 'from' }) + public from!: string; + + @Column({ name: 'to' }) + public to!: string; + + @Column({ name: 'value', type: 'bigint', transformer: bigNumberTransformer }) + public value!: BigNumber; + + @Column({ name: 'gas', type: 'bigint', transformer: bigNumberTransformer }) + public gas!: BigNumber; + + @Column({ name: 'gas_price', type: 'bigint', transformer: bigNumberTransformer }) + public gasPrice!: BigNumber; + + @Column({ name: 'is_error', type: 'boolean', nullable: true }) + public isError!: boolean; + + @Column({ name: 'txreceipt_status' }) + public txreceiptStatus?: string; + + @Column({ name: 'input' }) + public input!: string; + + @Column({ name: 'contract_address' }) + public contractAddress!: string; + + @Column({ name: 'cumulative_gas_used', type: 'bigint', transformer: bigNumberTransformer }) + public cumulativeGasUsed!: BigNumber; + + @Column({ name: 'gas_used', type: 'bigint', transformer: bigNumberTransformer }) + public gasUsed!: BigNumber; + + @Column({ name: 'confirmations', type: 'bigint', transformer: bigNumberTransformer }) + public confirmations!: BigNumber; + +} diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts index 174a32a446..0f3e85f9c6 100644 --- a/packages/pipeline/src/entities/index.ts +++ b/packages/pipeline/src/entities/index.ts @@ -4,6 +4,7 @@ import { ExchangeFillEvent } from './exchange_fill_event'; export { Block } from './block'; export { DexTrade } from './dex_trade'; +export { EtherscanTransaction } from './etherscan_transaction'; export { ExchangeCancelEvent } from './exchange_cancel_event'; export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event'; export { ExchangeFillEvent } from './exchange_fill_event'; diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 4604686acf..e2e6be7691 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -9,6 +9,7 @@ import { CopperOpportunity, DexTrade, ERC20ApprovalEvent, + EtherscanTransaction, ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, @@ -31,6 +32,7 @@ const entities = [ CopperCustomField, CopperLead, DexTrade, + EtherscanTransaction, ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, diff --git a/packages/pipeline/src/parsers/etherscan/index.ts b/packages/pipeline/src/parsers/etherscan/index.ts new file mode 100644 index 0000000000..1c13ebc739 --- /dev/null +++ b/packages/pipeline/src/parsers/etherscan/index.ts @@ -0,0 +1,42 @@ +import { BigNumber } from '@0x/utils'; +import * as R from 'ramda'; + +import { EtherscanTransactionResponse } from '../../data_sources/etherscan'; +import { EtherscanTransaction } from '../../entities'; + +/** + * Parses an Etherscan response from the Etherscan API and returns an array of + * EtherscanTransaction entities. + * @param rawTrades A raw order response from an SRA endpoint. + */ +export function parseEtherscanTransactions(rawTransactions: EtherscanTransactionResponse[]): EtherscanTransaction[] { + return R.map(_parseEtherscanTransaction, rawTransactions); +} + +/** + * Converts a single Etherscan transction into an EtherscanTransaction entity. + * @param rawTx A single Etherscan transaction from the Etherscan API. + */ +export function _parseEtherscanTransaction(rawTx: EtherscanTransactionResponse): EtherscanTransaction { + const parsedTx = new EtherscanTransaction(); + parsedTx.blockNumber = new BigNumber(rawTx.blockNumber); + parsedTx.timeStamp = new BigNumber(rawTx.timeStamp); + parsedTx.hash = rawTx.hash; + parsedTx.blockHash = rawTx.blockHash; + parsedTx.transactionIndex = Number(rawTx.transactionIndex); + parsedTx.nonce = Number(rawTx.nonce); + parsedTx.from = rawTx.from; + parsedTx.to = rawTx.to; + parsedTx.value = new BigNumber(rawTx.value); + parsedTx.gas = new BigNumber(rawTx.gas); + parsedTx.gasPrice = new BigNumber(rawTx.gasPrice); + parsedTx.isError = rawTx.isError === '0' ? false : true; + parsedTx.txreceiptStatus = rawTx.txreceiptStatus; + parsedTx.input = rawTx.input; + parsedTx.contractAddress = rawTx.contractAddress; + parsedTx.cumulativeGasUsed = new BigNumber(rawTx.cumulativeGasUsed); + parsedTx.gasUsed = new BigNumber(rawTx.gasUsed); + parsedTx.confirmations = new BigNumber(rawTx.confirmations); + + return parsedTx; +} diff --git a/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts b/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts new file mode 100644 index 0000000000..ac1d0eb8fd --- /dev/null +++ b/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts @@ -0,0 +1,68 @@ +import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm'; + +import { getContractAddressesForNetworkOrThrow, NetworkId } from '@0x/contract-addresses'; +import { web3Factory } from '@0x/dev-utils'; +import { Web3ProviderEngine } from '@0x/subproviders'; +import { logUtils } from '@0x/utils'; +import { Web3Wrapper } from '@0x/web3-wrapper'; + +import { EtherscanSource } from '../data_sources/etherscan'; +import { EtherscanTransaction } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseEtherscanTransactions } from '../parsers/etherscan'; +import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils'; + +const BATCH_SAVE_SIZE = 1000; // Number of orders to save at once. +const START_BLOCK_OFFSET = 100; // Number of blocks before the last known block to consider when updating fill events. +const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default endBlock. + +// API key to use if environment variable has not been set +const FALLBACK_API_KEY = 'YourApiKeyToken'; + +let connection: Connection; + +(async () => { + let apiKey = process.env.ETHERSCAN_API_KEY; + if (apiKey === undefined) { + logUtils.log(`Missing env var: ETHERSCAN_API_KEY - using default API key: ${FALLBACK_API_KEY}`); + apiKey = FALLBACK_API_KEY; + } + connection = await createConnection(ormConfig as ConnectionOptions); + const provider = web3Factory.getRpcProvider({ + rpcUrl: INFURA_ROOT_URL, + }); + const EtherscanTransactionRepository = connection.getRepository(EtherscanTransaction); + const startBlock = await getStartBlockAsync(EtherscanTransactionRepository); + logUtils.log(`Start block: ${startBlock}`); + const endBlock = await calculateEndBlockAsync(provider); + logUtils.log(`End block: ${endBlock}`); + const etherscanSource = new EtherscanSource(apiKey, startBlock, endBlock); + const exchangeContractAddress = getContractAddressesForNetworkOrThrow(NetworkId.Mainnet).exchange; + logUtils.log('Fetching exchange transactions from Etherscan...'); + const rawTransactions = await etherscanSource.getEtherscanTransactionsForAddressAsync(exchangeContractAddress); + const transactions = parseEtherscanTransactions(rawTransactions); + logUtils.log(`Saving ${transactions.length} records to database`); + await EtherscanTransactionRepository.save(transactions, { chunk: Math.ceil(transactions.length / BATCH_SAVE_SIZE) }); + logUtils.log('Done'); + process.exit(0); +})().catch(handleError); + +async function getStartBlockAsync(repository: Repository): Promise { + const transactionsCount = await repository.count(); + if (transactionsCount === 0) { + logUtils.log(`No existing ${repository.metadata.name}s found.`); + return EXCHANGE_START_BLOCK; + } + const tableName = repository.metadata.tableName; + const queryResult = await connection.query( + `SELECT block_number FROM raw.${tableName} ORDER BY block_number DESC LIMIT 1`, + ); + const lastKnownBlock = queryResult[0].block_number; + return lastKnownBlock - START_BLOCK_OFFSET; +} + +async function calculateEndBlockAsync(provider: Web3ProviderEngine): Promise { + const web3Wrapper = new Web3Wrapper(provider); + const currentBlock = await web3Wrapper.getBlockNumberAsync(); + return currentBlock - BLOCK_FINALITY_THRESHOLD; +} diff --git a/packages/pipeline/test/entities/etherscan_test.ts b/packages/pipeline/test/entities/etherscan_test.ts new file mode 100644 index 0000000000..7b35d92b78 --- /dev/null +++ b/packages/pipeline/test/entities/etherscan_test.ts @@ -0,0 +1,44 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; +import 'reflect-metadata'; + +import { EtherscanTransaction } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const transaction: EtherscanTransaction = { + blockNumber: new BigNumber('6271590'), + timeStamp: new BigNumber('1536083185'), + hash: '0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea', + nonce: 2, + blockHash: '0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee', + transactionIndex: 3, + from: '0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713', + to: '', + value: new BigNumber('0'), + gas: new BigNumber('7000000'), + gasPrice: new BigNumber('20000000000'), + isError: false, + txreceiptStatus: '1', + input: '0x60806040', // shortened + contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + cumulativeGasUsed: new BigNumber('6068925'), + gasUsed: new BigNumber('6005925'), + confirmations: new BigNumber('976529'), +}; + +// tslint:disable:custom-no-magic-numbers +describe('EtherscanTransaction entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const transactions = [transaction]; + const tradesRepository = connection.getRepository(EtherscanTransaction); + for (const tx of transactions) { + await testSaveAndFindEntityAsync(tradesRepository, tx); + } + }); +}); diff --git a/packages/pipeline/test/parsers/etherscan/index_test.ts b/packages/pipeline/test/parsers/etherscan/index_test.ts new file mode 100644 index 0000000000..fc118a2c92 --- /dev/null +++ b/packages/pipeline/test/parsers/etherscan/index_test.ts @@ -0,0 +1,63 @@ +import { BigNumber } from '@0x/utils'; +import * as chai from 'chai'; +import 'mocha'; + +import { EtherscanTransactionResponse } from '../../../src/data_sources/etherscan'; +import { EtherscanTransaction } from '../../../src/entities'; +import { parseEtherscanTransactions } from '../../../src/parsers/etherscan'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('etherscan_transactions', () => { + describe('parseEtherscanTransactions', () => { + it('converts etherscanTransactions to EtherscanTransaction entities', () => { + const response: EtherscanTransactionResponse[] = [{ + blockNumber: '6271590', + timeStamp: '1536083185', + hash: '0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea', + nonce: '2', + blockHash: '0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee', + transactionIndex: '3', + from: '0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713', + to: '', + value: '0', + gas: '7000000', + gasPrice: '20000000000', + isError: '0', + txreceiptStatus: '1', + input: '0x60806040', // shortened + contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + cumulativeGasUsed: '6068925', + gasUsed: '6005925', + confirmations: '976529', + }]; + + const _expected: EtherscanTransaction = { + blockNumber: new BigNumber('6271590'), + timeStamp: new BigNumber('1536083185'), + hash: '0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea', + nonce: 2, + blockHash: '0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee', + transactionIndex: 3, + from: '0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713', + to: '', + value: new BigNumber('0'), + gas: new BigNumber('7000000'), + gasPrice: new BigNumber('20000000000'), + isError: false, + txreceiptStatus: '1', + input: '0x60806040', // shortened + contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + cumulativeGasUsed: new BigNumber('6068925'), + gasUsed: new BigNumber('6005925'), + confirmations: new BigNumber('976529'), + }; + const expected = [_expected]; + const actual = parseEtherscanTransactions(response); + expect(actual).deep.equal(expected); + }); + }); +}); From bf1115d4176805d495ff572ac82bf9d4087985cc Mon Sep 17 00:00:00 2001 From: askeluv Date: Fri, 22 Feb 2019 09:45:25 +0800 Subject: [PATCH 2/6] Backfills automatically + prettier --- ...50749543417-CreateEtherscanTransactions.ts | 6 +-- .../src/data_sources/etherscan/index.ts | 3 +- .../src/entities/etherscan_transaction.ts | 1 - ...ll_exchange_transactions_from_etherscan.ts | 4 +- .../test/parsers/etherscan/index_test.ts | 42 ++++++++++--------- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/packages/pipeline/migrations/1550749543417-CreateEtherscanTransactions.ts b/packages/pipeline/migrations/1550749543417-CreateEtherscanTransactions.ts index b35f3a6e45..4b0dbd7075 100644 --- a/packages/pipeline/migrations/1550749543417-CreateEtherscanTransactions.ts +++ b/packages/pipeline/migrations/1550749543417-CreateEtherscanTransactions.ts @@ -1,10 +1,10 @@ -import { MigrationInterface, QueryRunner, Table } from "typeorm"; +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; const table = new Table({ name: 'raw.etherscan_transactions', columns: [ { name: 'hash', type: 'varchar', isPrimary: true }, - + { name: 'block_number', type: 'numeric', isNullable: false }, { name: 'timestamp', type: 'numeric', isNullable: false }, { name: 'block_hash', type: 'varchar', isNullable: false }, @@ -22,12 +22,10 @@ const table = new Table({ { name: 'cumulative_gas_used', type: 'numeric', isNullable: false }, { name: 'gas_used', type: 'numeric', isNullable: false }, { name: 'confirmations', type: 'numeric', isNullable: false }, - ], }); export class CreateEtherscanTransactions1550749543417 implements MigrationInterface { - public async up(queryRunner: QueryRunner): Promise { await queryRunner.createTable(table); } diff --git a/packages/pipeline/src/data_sources/etherscan/index.ts b/packages/pipeline/src/data_sources/etherscan/index.ts index 96a5266a92..02d381af84 100644 --- a/packages/pipeline/src/data_sources/etherscan/index.ts +++ b/packages/pipeline/src/data_sources/etherscan/index.ts @@ -32,11 +32,10 @@ export interface EtherscanTransactionResponse { // tslint:disable:prefer-function-over-method // ^ Keep consistency with other sources and help logical organization export class EtherscanSource { - public readonly _urlBase: string; constructor(apiKey: string, startBlock: number, endBlock: number) { - this._urlBase = `http://api.etherscan.io/api?module=account&action=txlist&startblock=${startBlock}&endblock=${endBlock}&sort=desc&apikey=${apiKey}&address=`; + this._urlBase = `http://api.etherscan.io/api?module=account&action=txlist&startblock=${startBlock}&endblock=${endBlock}&sort=asc&apikey=${apiKey}&address=`; } /** diff --git a/packages/pipeline/src/entities/etherscan_transaction.ts b/packages/pipeline/src/entities/etherscan_transaction.ts index 3de6f4977d..c84eddc978 100644 --- a/packages/pipeline/src/entities/etherscan_transaction.ts +++ b/packages/pipeline/src/entities/etherscan_transaction.ts @@ -58,5 +58,4 @@ export class EtherscanTransaction { @Column({ name: 'confirmations', type: 'bigint', transformer: bigNumberTransformer }) public confirmations!: BigNumber; - } diff --git a/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts b/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts index ac1d0eb8fd..6c93e1ae18 100644 --- a/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts +++ b/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts @@ -42,7 +42,9 @@ let connection: Connection; const rawTransactions = await etherscanSource.getEtherscanTransactionsForAddressAsync(exchangeContractAddress); const transactions = parseEtherscanTransactions(rawTransactions); logUtils.log(`Saving ${transactions.length} records to database`); - await EtherscanTransactionRepository.save(transactions, { chunk: Math.ceil(transactions.length / BATCH_SAVE_SIZE) }); + await EtherscanTransactionRepository.save(transactions, { + chunk: Math.ceil(transactions.length / BATCH_SAVE_SIZE), + }); logUtils.log('Done'); process.exit(0); })().catch(handleError); diff --git a/packages/pipeline/test/parsers/etherscan/index_test.ts b/packages/pipeline/test/parsers/etherscan/index_test.ts index fc118a2c92..bf7aab4e5a 100644 --- a/packages/pipeline/test/parsers/etherscan/index_test.ts +++ b/packages/pipeline/test/parsers/etherscan/index_test.ts @@ -14,26 +14,28 @@ const expect = chai.expect; describe('etherscan_transactions', () => { describe('parseEtherscanTransactions', () => { it('converts etherscanTransactions to EtherscanTransaction entities', () => { - const response: EtherscanTransactionResponse[] = [{ - blockNumber: '6271590', - timeStamp: '1536083185', - hash: '0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea', - nonce: '2', - blockHash: '0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee', - transactionIndex: '3', - from: '0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713', - to: '', - value: '0', - gas: '7000000', - gasPrice: '20000000000', - isError: '0', - txreceiptStatus: '1', - input: '0x60806040', // shortened - contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', - cumulativeGasUsed: '6068925', - gasUsed: '6005925', - confirmations: '976529', - }]; + const response: EtherscanTransactionResponse[] = [ + { + blockNumber: '6271590', + timeStamp: '1536083185', + hash: '0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea', + nonce: '2', + blockHash: '0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee', + transactionIndex: '3', + from: '0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713', + to: '', + value: '0', + gas: '7000000', + gasPrice: '20000000000', + isError: '0', + txreceiptStatus: '1', + input: '0x60806040', // shortened + contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + cumulativeGasUsed: '6068925', + gasUsed: '6005925', + confirmations: '976529', + }, + ]; const _expected: EtherscanTransaction = { blockNumber: new BigNumber('6271590'), From 6eb923d22f248de01e0032f25a269a734383b3ef Mon Sep 17 00:00:00 2001 From: askeluv Date: Fri, 22 Feb 2019 11:21:33 +0800 Subject: [PATCH 3/6] Throw error when API key is missing + use response in parser test --- .../pull_exchange_transactions_from_etherscan.ts | 8 ++------ .../pipeline/test/parsers/etherscan/index_test.ts | 12 ++++++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts b/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts index 6c93e1ae18..23c8cf4079 100644 --- a/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts +++ b/packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts @@ -16,16 +16,12 @@ const BATCH_SAVE_SIZE = 1000; // Number of orders to save at once. const START_BLOCK_OFFSET = 100; // Number of blocks before the last known block to consider when updating fill events. const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default endBlock. -// API key to use if environment variable has not been set -const FALLBACK_API_KEY = 'YourApiKeyToken'; - let connection: Connection; (async () => { - let apiKey = process.env.ETHERSCAN_API_KEY; + const apiKey = process.env.ETHERSCAN_API_KEY; if (apiKey === undefined) { - logUtils.log(`Missing env var: ETHERSCAN_API_KEY - using default API key: ${FALLBACK_API_KEY}`); - apiKey = FALLBACK_API_KEY; + throw new Error('Missing required env var: ETHERSCAN_API_KEY'); } connection = await createConnection(ormConfig as ConnectionOptions); const provider = web3Factory.getRpcProvider({ diff --git a/packages/pipeline/test/parsers/etherscan/index_test.ts b/packages/pipeline/test/parsers/etherscan/index_test.ts index bf7aab4e5a..f0626cdc92 100644 --- a/packages/pipeline/test/parsers/etherscan/index_test.ts +++ b/packages/pipeline/test/parsers/etherscan/index_test.ts @@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils'; import * as chai from 'chai'; import 'mocha'; -import { EtherscanTransactionResponse } from '../../../src/data_sources/etherscan'; +import { EtherscanResponse, EtherscanTransactionResponse } from '../../../src/data_sources/etherscan'; import { EtherscanTransaction } from '../../../src/entities'; import { parseEtherscanTransactions } from '../../../src/parsers/etherscan'; import { chaiSetup } from '../../utils/chai_setup'; @@ -14,7 +14,7 @@ const expect = chai.expect; describe('etherscan_transactions', () => { describe('parseEtherscanTransactions', () => { it('converts etherscanTransactions to EtherscanTransaction entities', () => { - const response: EtherscanTransactionResponse[] = [ + const result: EtherscanTransactionResponse[] = [ { blockNumber: '6271590', timeStamp: '1536083185', @@ -36,7 +36,11 @@ describe('etherscan_transactions', () => { confirmations: '976529', }, ]; - + const response: EtherscanResponse = { + status: '1', + message: 'OK', + result, + }; const _expected: EtherscanTransaction = { blockNumber: new BigNumber('6271590'), timeStamp: new BigNumber('1536083185'), @@ -58,7 +62,7 @@ describe('etherscan_transactions', () => { confirmations: new BigNumber('976529'), }; const expected = [_expected]; - const actual = parseEtherscanTransactions(response); + const actual = parseEtherscanTransactions(response.result); expect(actual).deep.equal(expected); }); }); From 9fbd809344c4c9b1ce97e8e13abd7aeb9c3b124b Mon Sep 17 00:00:00 2001 From: askeluv Date: Tue, 26 Feb 2019 11:42:53 +0800 Subject: [PATCH 4/6] Improved testing - added fixtures --- .../src/data_sources/etherscan/index.ts | 2 +- .../pipeline/src/parsers/etherscan/index.ts | 2 +- .../api_v1_accounts_transactions.json | 26 +++++++++ .../etherscan/api_v1_accounts_transactions.ts | 30 +++++++++++ .../test/parsers/etherscan/index_test.ts | 53 ++----------------- packages/pipeline/tsconfig.json | 3 +- 6 files changed, 65 insertions(+), 51 deletions(-) create mode 100644 packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.json create mode 100644 packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.ts diff --git a/packages/pipeline/src/data_sources/etherscan/index.ts b/packages/pipeline/src/data_sources/etherscan/index.ts index 02d381af84..0c260a2776 100644 --- a/packages/pipeline/src/data_sources/etherscan/index.ts +++ b/packages/pipeline/src/data_sources/etherscan/index.ts @@ -21,7 +21,7 @@ export interface EtherscanTransactionResponse { gas: string; gasPrice: string; isError: string; - txreceiptStatus: string; + txreceipt_status: string; input: string; contractAddress: string; cumulativeGasUsed: string; diff --git a/packages/pipeline/src/parsers/etherscan/index.ts b/packages/pipeline/src/parsers/etherscan/index.ts index 1c13ebc739..24ecf0445c 100644 --- a/packages/pipeline/src/parsers/etherscan/index.ts +++ b/packages/pipeline/src/parsers/etherscan/index.ts @@ -31,7 +31,7 @@ export function _parseEtherscanTransaction(rawTx: EtherscanTransactionResponse): parsedTx.gas = new BigNumber(rawTx.gas); parsedTx.gasPrice = new BigNumber(rawTx.gasPrice); parsedTx.isError = rawTx.isError === '0' ? false : true; - parsedTx.txreceiptStatus = rawTx.txreceiptStatus; + parsedTx.txreceiptStatus = rawTx.txreceipt_status; parsedTx.input = rawTx.input; parsedTx.contractAddress = rawTx.contractAddress; parsedTx.cumulativeGasUsed = new BigNumber(rawTx.cumulativeGasUsed); diff --git a/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.json b/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.json new file mode 100644 index 0000000000..fb2fe1b95d --- /dev/null +++ b/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.json @@ -0,0 +1,26 @@ +{ + "message": "OK", + "result": [ + { + "blockHash": "0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee", + "blockNumber": "6271590", + "confirmations": "996941", + "contractAddress": "0x4f833a24e1f95d70f028921e27040ca56e09ab0b", + "cumulativeGasUsed": "6068925", + "from": "0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713", + "gas": "7000000", + "gasPrice": "20000000000", + "gasUsed": "6005925", + "hash": "0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea", + "input": "0x60806040", + "isError": "0", + "nonce": "2", + "timeStamp": "1536083185", + "to": "", + "transactionIndex": "3", + "txreceipt_status": "1", + "value": "0" + } + ], + "status": "1" +} diff --git a/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.ts b/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.ts new file mode 100644 index 0000000000..f191b40b63 --- /dev/null +++ b/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.ts @@ -0,0 +1,30 @@ +import { BigNumber } from '@0x/utils'; + +import { EtherscanTransaction } from '../../../src/entities'; + +// To re-create the JSON file from the API (e.g. if the API output schema changes), run the below command: +// curl "https://api.etherscan.io/api?module=account&action=txlist&address=0x4f833a24e1f95d70f028921e27040ca56e09ab0b&startblock=0&endblock=99999999&page=1&offset=1&sort=asc&apikey=YourApiKeyToken" | python -m json.tool > api_v1_accounts_transactions.json + +const ParsedEtherscanTransactions: EtherscanTransaction[] = [ + { + blockNumber: new BigNumber('6271590'), + timeStamp: new BigNumber('1536083185'), + hash: '0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea', + nonce: 2, + blockHash: '0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee', + transactionIndex: 3, + from: '0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713', + to: '', + value: new BigNumber('0'), + gas: new BigNumber('7000000'), + gasPrice: new BigNumber('20000000000'), + isError: false, + txreceiptStatus: '1', + input: '0x60806040', // shortened + contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + cumulativeGasUsed: new BigNumber('6068925'), + gasUsed: new BigNumber('6005925'), + confirmations: new BigNumber('996941'), + } +]; +export { ParsedEtherscanTransactions }; diff --git a/packages/pipeline/test/parsers/etherscan/index_test.ts b/packages/pipeline/test/parsers/etherscan/index_test.ts index f0626cdc92..172464971c 100644 --- a/packages/pipeline/test/parsers/etherscan/index_test.ts +++ b/packages/pipeline/test/parsers/etherscan/index_test.ts @@ -7,6 +7,9 @@ import { EtherscanTransaction } from '../../../src/entities'; import { parseEtherscanTransactions } from '../../../src/parsers/etherscan'; import { chaiSetup } from '../../utils/chai_setup'; +import * as etherscanResponse from '../../fixtures/etherscan/api_v1_accounts_transactions.json'; +import { ParsedEtherscanTransactions } from '../../fixtures/etherscan/api_v1_accounts_transactions'; + chaiSetup.configure(); const expect = chai.expect; @@ -14,54 +17,8 @@ const expect = chai.expect; describe('etherscan_transactions', () => { describe('parseEtherscanTransactions', () => { it('converts etherscanTransactions to EtherscanTransaction entities', () => { - const result: EtherscanTransactionResponse[] = [ - { - blockNumber: '6271590', - timeStamp: '1536083185', - hash: '0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea', - nonce: '2', - blockHash: '0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee', - transactionIndex: '3', - from: '0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713', - to: '', - value: '0', - gas: '7000000', - gasPrice: '20000000000', - isError: '0', - txreceiptStatus: '1', - input: '0x60806040', // shortened - contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', - cumulativeGasUsed: '6068925', - gasUsed: '6005925', - confirmations: '976529', - }, - ]; - const response: EtherscanResponse = { - status: '1', - message: 'OK', - result, - }; - const _expected: EtherscanTransaction = { - blockNumber: new BigNumber('6271590'), - timeStamp: new BigNumber('1536083185'), - hash: '0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea', - nonce: 2, - blockHash: '0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee', - transactionIndex: 3, - from: '0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713', - to: '', - value: new BigNumber('0'), - gas: new BigNumber('7000000'), - gasPrice: new BigNumber('20000000000'), - isError: false, - txreceiptStatus: '1', - input: '0x60806040', // shortened - contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', - cumulativeGasUsed: new BigNumber('6068925'), - gasUsed: new BigNumber('6005925'), - confirmations: new BigNumber('976529'), - }; - const expected = [_expected]; + const response: EtherscanResponse = etherscanResponse; + const expected = ParsedEtherscanTransactions; const actual = parseEtherscanTransactions(response.result); expect(actual).deep.equal(expected); }); diff --git a/packages/pipeline/tsconfig.json b/packages/pipeline/tsconfig.json index 45e07374c4..4c43775d74 100644 --- a/packages/pipeline/tsconfig.json +++ b/packages/pipeline/tsconfig.json @@ -13,6 +13,7 @@ "./test/fixtures/copper/api_v1_custom_field_definitions.json", "./test/fixtures/copper/api_v1_list_activities.json", "./test/fixtures/copper/api_v1_list_leads.json", - "./test/fixtures/copper/api_v1_list_opportunities.json" + "./test/fixtures/copper/api_v1_list_opportunities.json", + "./test/fixtures/etherscan/api_v1_accounts_transactions.json" ] } From 6ac9e11245d957b6f25eb1d859634a0cfb4b6e25 Mon Sep 17 00:00:00 2001 From: askeluv Date: Tue, 26 Feb 2019 11:46:21 +0800 Subject: [PATCH 5/6] Linting --- .../test/fixtures/etherscan/api_v1_accounts_transactions.ts | 2 +- packages/pipeline/test/parsers/etherscan/index_test.ts | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.ts b/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.ts index f191b40b63..95a81d8833 100644 --- a/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.ts +++ b/packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.ts @@ -25,6 +25,6 @@ const ParsedEtherscanTransactions: EtherscanTransaction[] = [ cumulativeGasUsed: new BigNumber('6068925'), gasUsed: new BigNumber('6005925'), confirmations: new BigNumber('996941'), - } + }, ]; export { ParsedEtherscanTransactions }; diff --git a/packages/pipeline/test/parsers/etherscan/index_test.ts b/packages/pipeline/test/parsers/etherscan/index_test.ts index 172464971c..8c021299a1 100644 --- a/packages/pipeline/test/parsers/etherscan/index_test.ts +++ b/packages/pipeline/test/parsers/etherscan/index_test.ts @@ -1,14 +1,12 @@ -import { BigNumber } from '@0x/utils'; import * as chai from 'chai'; import 'mocha'; -import { EtherscanResponse, EtherscanTransactionResponse } from '../../../src/data_sources/etherscan'; -import { EtherscanTransaction } from '../../../src/entities'; +import { EtherscanResponse } from '../../../src/data_sources/etherscan'; import { parseEtherscanTransactions } from '../../../src/parsers/etherscan'; import { chaiSetup } from '../../utils/chai_setup'; -import * as etherscanResponse from '../../fixtures/etherscan/api_v1_accounts_transactions.json'; import { ParsedEtherscanTransactions } from '../../fixtures/etherscan/api_v1_accounts_transactions'; +import * as etherscanResponse from '../../fixtures/etherscan/api_v1_accounts_transactions.json'; chaiSetup.configure(); const expect = chai.expect; From 85ed5d27f5fcd726083fd9aaa5e78b33ed53cf75 Mon Sep 17 00:00:00 2001 From: askeluv Date: Tue, 26 Feb 2019 19:37:59 +0800 Subject: [PATCH 6/6] Bumped up timeout to 4 mins --- packages/pipeline/src/data_sources/etherscan/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pipeline/src/data_sources/etherscan/index.ts b/packages/pipeline/src/data_sources/etherscan/index.ts index 0c260a2776..8a0746768a 100644 --- a/packages/pipeline/src/data_sources/etherscan/index.ts +++ b/packages/pipeline/src/data_sources/etherscan/index.ts @@ -1,6 +1,6 @@ import { fetchAsync } from '@0x/utils'; -const TIMEOUT = 120000; +const TIMEOUT = 240000; export interface EtherscanResponse { status: string;