Merge pull request #1629 from 0xProject/etherscan-exchange-transactions

Pull exchange contract transactions from Etherscan

packages/pipeline/migrations/1550749543417-CreateEtherscanTransactions.ts

@@ -0,0 +1,36 @@
+import { MigrationInterface, QueryRunner, Table } from 'typeorm';
+
+const table = new Table({
+    name: 'raw.etherscan_transactions',
+    columns: [
+        { name: 'hash', type: 'varchar', isPrimary: true },
+
+        { name: 'block_number', type: 'numeric', isNullable: false },
+        { name: 'timestamp', type: 'numeric', isNullable: false },
+        { name: 'block_hash', type: 'varchar', isNullable: false },
+        { name: 'transaction_index', type: 'numeric', isNullable: false },
+        { name: 'nonce', type: 'numeric', isNullable: false },
+        { name: 'from', type: 'varchar', isNullable: false },
+        { name: 'to', type: 'varchar', isNullable: false },
+        { name: 'value', type: 'numeric', isNullable: false },
+        { name: 'gas', type: 'numeric', isNullable: false },
+        { name: 'gas_price', type: 'numeric', isNullable: false },
+        { name: 'is_error', type: 'boolean', isNullable: false },
+        { name: 'txreceipt_status', type: 'varchar', isNullable: true },
+        { name: 'input', type: 'varchar', isNullable: false },
+        { name: 'contract_address', type: 'varchar', isNullable: false },
+        { name: 'cumulative_gas_used', type: 'numeric', isNullable: false },
+        { name: 'gas_used', type: 'numeric', isNullable: false },
+        { name: 'confirmations', type: 'numeric', isNullable: false },
+    ],
+});
+
+export class CreateEtherscanTransactions1550749543417 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.createTable(table);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.dropTable(table);
+    }
+}

packages/pipeline/src/data_sources/etherscan/index.ts

@@ -0,0 +1,50 @@
+import { fetchAsync } from '@0x/utils';
+
+const TIMEOUT = 240000;
+
+export interface EtherscanResponse {
+    status: string;
+    message: string;
+    result: EtherscanTransactionResponse[];
+}
+
+export interface EtherscanTransactionResponse {
+    blockNumber: string;
+    timeStamp: string;
+    hash: string;
+    nonce: string;
+    blockHash: string;
+    transactionIndex: string;
+    from: string;
+    to: string;
+    value: string;
+    gas: string;
+    gasPrice: string;
+    isError: string;
+    txreceipt_status: string;
+    input: string;
+    contractAddress: string;
+    cumulativeGasUsed: string;
+    gasUsed: string;
+    confirmations: string;
+}
+
+// tslint:disable:prefer-function-over-method
+// ^ Keep consistency with other sources and help logical organization
+export class EtherscanSource {
+    public readonly _urlBase: string;
+
+    constructor(apiKey: string, startBlock: number, endBlock: number) {
+        this._urlBase = `http://api.etherscan.io/api?module=account&action=txlist&startblock=${startBlock}&endblock=${endBlock}&sort=asc&apikey=${apiKey}&address=`;
+    }
+
+    /**
+     * Call Etherscan API and return result.
+     */
+    public async getEtherscanTransactionsForAddressAsync(address: string): Promise<EtherscanTransactionResponse[]> {
+        const urlWithAddress = this._urlBase + address;
+        const resp = await fetchAsync(urlWithAddress, {}, TIMEOUT);
+        const respJson: EtherscanResponse = await resp.json();
+        return respJson.result;
+    }
+}

packages/pipeline/src/entities/etherscan_transaction.ts

@@ -0,0 +1,61 @@
+import { BigNumber } from '@0x/utils';
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { bigNumberTransformer, numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'etherscan_transactions', schema: 'raw' })
+export class EtherscanTransaction {
+    @PrimaryColumn({ name: 'hash' })
+    public hash!: string;
+
+    @Column({ name: 'block_number', type: 'bigint', transformer: bigNumberTransformer })
+    public blockNumber!: BigNumber;
+
+    @Column({ name: 'timestamp', type: 'bigint', transformer: bigNumberTransformer })
+    public timeStamp!: BigNumber;
+
+    @Column({ name: 'block_hash' })
+    public blockHash!: string;
+
+    @Column({ name: 'transaction_index', transformer: numberToBigIntTransformer })
+    public transactionIndex!: number;
+
+    @Column({ name: 'nonce', transformer: numberToBigIntTransformer })
+    public nonce!: number;
+
+    @Column({ name: 'from' })
+    public from!: string;
+
+    @Column({ name: 'to' })
+    public to!: string;
+
+    @Column({ name: 'value', type: 'bigint', transformer: bigNumberTransformer })
+    public value!: BigNumber;
+
+    @Column({ name: 'gas', type: 'bigint', transformer: bigNumberTransformer })
+    public gas!: BigNumber;
+
+    @Column({ name: 'gas_price', type: 'bigint', transformer: bigNumberTransformer })
+    public gasPrice!: BigNumber;
+
+    @Column({ name: 'is_error', type: 'boolean', nullable: true })
+    public isError!: boolean;
+
+    @Column({ name: 'txreceipt_status' })
+    public txreceiptStatus?: string;
+
+    @Column({ name: 'input' })
+    public input!: string;
+
+    @Column({ name: 'contract_address' })
+    public contractAddress!: string;
+
+    @Column({ name: 'cumulative_gas_used', type: 'bigint', transformer: bigNumberTransformer })
+    public cumulativeGasUsed!: BigNumber;
+
+    @Column({ name: 'gas_used', type: 'bigint', transformer: bigNumberTransformer })
+    public gasUsed!: BigNumber;
+
+    @Column({ name: 'confirmations', type: 'bigint', transformer: bigNumberTransformer })
+    public confirmations!: BigNumber;
+}

packages/pipeline/src/entities/index.ts

@@ -4,6 +4,7 @@ import { ExchangeFillEvent } from './exchange_fill_event';
 
 export { Block } from './block';
 export { DexTrade } from './dex_trade';
+export { EtherscanTransaction } from './etherscan_transaction';
 export { ExchangeCancelEvent } from './exchange_cancel_event';
 export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event';
 export { ExchangeFillEvent } from './exchange_fill_event';

packages/pipeline/src/ormconfig.ts

@@ -9,6 +9,7 @@ import {
     CopperOpportunity,
     DexTrade,
     ERC20ApprovalEvent,
+    EtherscanTransaction,
     ExchangeCancelEvent,
     ExchangeCancelUpToEvent,
     ExchangeFillEvent,
@@ -31,6 +32,7 @@ const entities = [
     CopperCustomField,
     CopperLead,
     DexTrade,
+    EtherscanTransaction,
     ExchangeCancelEvent,
     ExchangeCancelUpToEvent,
     ExchangeFillEvent,

packages/pipeline/src/parsers/etherscan/index.ts

@@ -0,0 +1,42 @@
+import { BigNumber } from '@0x/utils';
+import * as R from 'ramda';
+
+import { EtherscanTransactionResponse } from '../../data_sources/etherscan';
+import { EtherscanTransaction } from '../../entities';
+
+/**
+ * Parses an Etherscan response from the Etherscan API and returns an array of
+ * EtherscanTransaction entities.
+ * @param rawTrades A raw order response from an SRA endpoint.
+ */
+export function parseEtherscanTransactions(rawTransactions: EtherscanTransactionResponse[]): EtherscanTransaction[] {
+    return R.map(_parseEtherscanTransaction, rawTransactions);
+}
+
+/**
+ * Converts a single Etherscan transction into an EtherscanTransaction entity.
+ * @param rawTx A single Etherscan transaction from the Etherscan API.
+ */
+export function _parseEtherscanTransaction(rawTx: EtherscanTransactionResponse): EtherscanTransaction {
+    const parsedTx = new EtherscanTransaction();
+    parsedTx.blockNumber = new BigNumber(rawTx.blockNumber);
+    parsedTx.timeStamp = new BigNumber(rawTx.timeStamp);
+    parsedTx.hash = rawTx.hash;
+    parsedTx.blockHash = rawTx.blockHash;
+    parsedTx.transactionIndex = Number(rawTx.transactionIndex);
+    parsedTx.nonce = Number(rawTx.nonce);
+    parsedTx.from = rawTx.from;
+    parsedTx.to = rawTx.to;
+    parsedTx.value = new BigNumber(rawTx.value);
+    parsedTx.gas = new BigNumber(rawTx.gas);
+    parsedTx.gasPrice = new BigNumber(rawTx.gasPrice);
+    parsedTx.isError = rawTx.isError === '0' ? false : true;
+    parsedTx.txreceiptStatus = rawTx.txreceipt_status;
+    parsedTx.input = rawTx.input;
+    parsedTx.contractAddress = rawTx.contractAddress;
+    parsedTx.cumulativeGasUsed = new BigNumber(rawTx.cumulativeGasUsed);
+    parsedTx.gasUsed = new BigNumber(rawTx.gasUsed);
+    parsedTx.confirmations = new BigNumber(rawTx.confirmations);
+
+    return parsedTx;
+}

packages/pipeline/src/scripts/pull_exchange_transactions_from_etherscan.ts

@@ -0,0 +1,66 @@
+import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm';
+
+import { getContractAddressesForNetworkOrThrow, NetworkId } from '@0x/contract-addresses';
+import { web3Factory } from '@0x/dev-utils';
+import { Web3ProviderEngine } from '@0x/subproviders';
+import { logUtils } from '@0x/utils';
+import { Web3Wrapper } from '@0x/web3-wrapper';
+
+import { EtherscanSource } from '../data_sources/etherscan';
+import { EtherscanTransaction } from '../entities';
+import * as ormConfig from '../ormconfig';
+import { parseEtherscanTransactions } from '../parsers/etherscan';
+import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils';
+
+const BATCH_SAVE_SIZE = 1000; // Number of orders to save at once.
+const START_BLOCK_OFFSET = 100; // Number of blocks before the last known block to consider when updating fill events.
+const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default endBlock.
+
+let connection: Connection;
+
+(async () => {
+    const apiKey = process.env.ETHERSCAN_API_KEY;
+    if (apiKey === undefined) {
+        throw new Error('Missing required env var: ETHERSCAN_API_KEY');
+    }
+    connection = await createConnection(ormConfig as ConnectionOptions);
+    const provider = web3Factory.getRpcProvider({
+        rpcUrl: INFURA_ROOT_URL,
+    });
+    const EtherscanTransactionRepository = connection.getRepository(EtherscanTransaction);
+    const startBlock = await getStartBlockAsync(EtherscanTransactionRepository);
+    logUtils.log(`Start block: ${startBlock}`);
+    const endBlock = await calculateEndBlockAsync(provider);
+    logUtils.log(`End block: ${endBlock}`);
+    const etherscanSource = new EtherscanSource(apiKey, startBlock, endBlock);
+    const exchangeContractAddress = getContractAddressesForNetworkOrThrow(NetworkId.Mainnet).exchange;
+    logUtils.log('Fetching exchange transactions from Etherscan...');
+    const rawTransactions = await etherscanSource.getEtherscanTransactionsForAddressAsync(exchangeContractAddress);
+    const transactions = parseEtherscanTransactions(rawTransactions);
+    logUtils.log(`Saving ${transactions.length} records to database`);
+    await EtherscanTransactionRepository.save(transactions, {
+        chunk: Math.ceil(transactions.length / BATCH_SAVE_SIZE),
+    });
+    logUtils.log('Done');
+    process.exit(0);
+})().catch(handleError);
+
+async function getStartBlockAsync<T>(repository: Repository<T>): Promise<number> {
+    const transactionsCount = await repository.count();
+    if (transactionsCount === 0) {
+        logUtils.log(`No existing ${repository.metadata.name}s found.`);
+        return EXCHANGE_START_BLOCK;
+    }
+    const tableName = repository.metadata.tableName;
+    const queryResult = await connection.query(
+        `SELECT block_number FROM raw.${tableName} ORDER BY block_number DESC LIMIT 1`,
+    );
+    const lastKnownBlock = queryResult[0].block_number;
+    return lastKnownBlock - START_BLOCK_OFFSET;
+}
+
+async function calculateEndBlockAsync(provider: Web3ProviderEngine): Promise<number> {
+    const web3Wrapper = new Web3Wrapper(provider);
+    const currentBlock = await web3Wrapper.getBlockNumberAsync();
+    return currentBlock - BLOCK_FINALITY_THRESHOLD;
+}

packages/pipeline/test/entities/etherscan_test.ts

@@ -0,0 +1,44 @@
+import { BigNumber } from '@0x/utils';
+import 'mocha';
+import 'reflect-metadata';
+
+import { EtherscanTransaction } from '../../src/entities';
+import { createDbConnectionOnceAsync } from '../db_setup';
+import { chaiSetup } from '../utils/chai_setup';
+
+import { testSaveAndFindEntityAsync } from './util';
+
+chaiSetup.configure();
+
+const transaction: EtherscanTransaction = {
+    blockNumber: new BigNumber('6271590'),
+    timeStamp: new BigNumber('1536083185'),
+    hash: '0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea',
+    nonce: 2,
+    blockHash: '0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee',
+    transactionIndex: 3,
+    from: '0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713',
+    to: '',
+    value: new BigNumber('0'),
+    gas: new BigNumber('7000000'),
+    gasPrice: new BigNumber('20000000000'),
+    isError: false,
+    txreceiptStatus: '1',
+    input: '0x60806040', // shortened
+    contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b',
+    cumulativeGasUsed: new BigNumber('6068925'),
+    gasUsed: new BigNumber('6005925'),
+    confirmations: new BigNumber('976529'),
+};
+
+// tslint:disable:custom-no-magic-numbers
+describe('EtherscanTransaction entity', () => {
+    it('save/find', async () => {
+        const connection = await createDbConnectionOnceAsync();
+        const transactions = [transaction];
+        const tradesRepository = connection.getRepository(EtherscanTransaction);
+        for (const tx of transactions) {
+            await testSaveAndFindEntityAsync(tradesRepository, tx);
+        }
+    });
+});

packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.json

@@ -0,0 +1,26 @@
+{
+    "message": "OK",
+    "result": [
+        {
+            "blockHash": "0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee",
+            "blockNumber": "6271590",
+            "confirmations": "996941",
+            "contractAddress": "0x4f833a24e1f95d70f028921e27040ca56e09ab0b",
+            "cumulativeGasUsed": "6068925",
+            "from": "0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713",
+            "gas": "7000000",
+            "gasPrice": "20000000000",
+            "gasUsed": "6005925",
+            "hash": "0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea",
+            "input": "0x60806040",
+            "isError": "0",
+            "nonce": "2",
+            "timeStamp": "1536083185",
+            "to": "",
+            "transactionIndex": "3",
+            "txreceipt_status": "1",
+            "value": "0"
+        }
+    ],
+    "status": "1"
+}

packages/pipeline/test/fixtures/etherscan/api_v1_accounts_transactions.ts

@@ -0,0 +1,30 @@
+import { BigNumber } from '@0x/utils';
+
+import { EtherscanTransaction } from '../../../src/entities';
+
+// To re-create the JSON file from the API (e.g. if the API output schema changes), run the below command:
+// curl "https://api.etherscan.io/api?module=account&action=txlist&address=0x4f833a24e1f95d70f028921e27040ca56e09ab0b&startblock=0&endblock=99999999&page=1&offset=1&sort=asc&apikey=YourApiKeyToken" | python -m json.tool > api_v1_accounts_transactions.json
+
+const ParsedEtherscanTransactions: EtherscanTransaction[] = [
+    {
+        blockNumber: new BigNumber('6271590'),
+        timeStamp: new BigNumber('1536083185'),
+        hash: '0x4a03044699c2fbd256e21632a6d8fbfc27655ea711157fa8b2b917f0eb954cea',
+        nonce: 2,
+        blockHash: '0xee634af4cebd034ed9e5e3dc873a2b0ecc60fe11bef27f7b92542388869f21ee',
+        transactionIndex: 3,
+        from: '0x2d7dc2ef7c6f6a2cbc3dba4db97b2ddb40e20713',
+        to: '',
+        value: new BigNumber('0'),
+        gas: new BigNumber('7000000'),
+        gasPrice: new BigNumber('20000000000'),
+        isError: false,
+        txreceiptStatus: '1',
+        input: '0x60806040', // shortened
+        contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b',
+        cumulativeGasUsed: new BigNumber('6068925'),
+        gasUsed: new BigNumber('6005925'),
+        confirmations: new BigNumber('996941'),
+    },
+];
+export { ParsedEtherscanTransactions };

packages/pipeline/test/parsers/etherscan/index_test.ts

@@ -0,0 +1,24 @@
+import * as chai from 'chai';
+import 'mocha';
+
+import { EtherscanResponse } from '../../../src/data_sources/etherscan';
+import { parseEtherscanTransactions } from '../../../src/parsers/etherscan';
+import { chaiSetup } from '../../utils/chai_setup';
+
+import { ParsedEtherscanTransactions } from '../../fixtures/etherscan/api_v1_accounts_transactions';
+import * as etherscanResponse from '../../fixtures/etherscan/api_v1_accounts_transactions.json';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('etherscan_transactions', () => {
+    describe('parseEtherscanTransactions', () => {
+        it('converts etherscanTransactions to EtherscanTransaction entities', () => {
+            const response: EtherscanResponse = etherscanResponse;
+            const expected = ParsedEtherscanTransactions;
+            const actual = parseEtherscanTransactions(response.result);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});

packages/pipeline/tsconfig.json

@@ -13,6 +13,7 @@
         "./test/fixtures/copper/api_v1_custom_field_definitions.json",
         "./test/fixtures/copper/api_v1_list_activities.json",
         "./test/fixtures/copper/api_v1_list_leads.json",
-        "./test/fixtures/copper/api_v1_list_opportunities.json"
+        "./test/fixtures/copper/api_v1_list_opportunities.json",
+        "./test/fixtures/etherscan/api_v1_accounts_transactions.json"
     ]
 }