Address PR feedback

packages/pipeline/README.md

@@ -42,7 +42,8 @@ Revert the most recent migration (CAUTION: may result in data loss!): `yarn migr
 
 There are several test scripts in **package.json**. You can run all the tests
 with `yarn test:all` or run certain tests seprately by following the
-instructions below. Some tests may not work out of the box on certain platforms.
+instructions below. Some tests may not work out of the box on certain platforms
+or operating systems (see the "Database tests" section below).
 
 ### Unit tests
 
@@ -71,8 +72,8 @@ Postgres is via Docker. Depending on your platform, you may need to prepend
 docker run --rm -d -p 5432:5432 --name pipeline_postgres postgres:11-alpine
 ```
 
-This will start a Postgres server with the default username and database name.
-You should set the environment variable as follows:
+This will start a Postgres server with the default username and database name
+(`postgres` and `postgres`). You should set the environment variable as follows:
 
 ```
 export ZEROEX_DATA_PIPELINE_DB_URL=postgresql://postgres@localhost/postgres
@@ -149,17 +150,17 @@ set the`ZEROEX_DATA_PIPELINE_DB_URL` environment variable to a valid
 
 #### Additional guidelines and tips:
 
-*   Table names should be plural and separated by underscores (e.g.,
+-   Table names should be plural and separated by underscores (e.g.,
     `exchange_fill_events`).
-*   Any table which contains data which comes directly from a third-party source
+-   Any table which contains data which comes directly from a third-party source
     should be namespaced in the `raw` PostgreSQL schema.
-*   Column names in the database should be separated by underscores (e.g.,
+-   Column names in the database should be separated by underscores (e.g.,
     `maker_asset_type`).
-*   Field names in entity classes (like any other fields in TypeScript) should
+-   Field names in entity classes (like any other fields in TypeScript) should
     be camel-cased (e.g., `makerAssetType`).
-*   All timestamps should be stored as milliseconds since the Unix Epoch.
-*   Use the `BigNumber` type for TypeScript code which deals with 256-bit
+-   All timestamps should be stored as milliseconds since the Unix Epoch.
+-   Use the `BigNumber` type for TypeScript code which deals with 256-bit
     numbers from smart contracts or for any case where we are dealing with large
     floating point numbers.
-*   [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a
+-   [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a
     helpful resource.

packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class ConvertTokenMetadataDecimalsToBigNumber1543980079179 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.query(
+            `ALTER TABLE raw.token_metadata
+                ALTER COLUMN decimals TYPE numeric USING decimals::numeric;`,
+        );
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.query(
+            `ALTER TABLE raw.token_metadata
+                ALTER COLUMN decimals TYPE numeric USING decimals::integer;`,
+        );
+    }
+}

packages/pipeline/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@0x/pipeline",
-    "version": "0.0.1",
+    "version": "1.0.0",
     "private": true,
     "description": "Data pipeline for offline analysis",
     "scripts": {
@@ -21,11 +21,6 @@
         "migrate:revert": "yarn typeorm migration:revert --config ./lib/src/ormconfig",
         "migrate:create": "yarn typeorm migration:create --config ./lib/src/ormconfig --dir migrations"
     },
-    "config": {
-        "postpublish": {
-            "assets": []
-        }
-    },
     "repository": {
         "type": "git",
         "url": "https://github.com/0xProject/0x-monorepo"

packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts

@@ -11,9 +11,10 @@ import { Web3ProviderEngine } from '@0x/subproviders';
 import { Web3Wrapper } from '@0x/web3-wrapper';
 import { LogWithDecodedArgs } from 'ethereum-types';
 
+import { EXCHANGE_START_BLOCK } from '../../utils';
+
 const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default toBlock.
 const NUM_BLOCKS_PER_QUERY = 20000; // Number of blocks to query for events at a time.
-const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
 
 export class ExchangeEventsSource {
     private readonly _exchangeWrapper: ExchangeWrapper;

packages/pipeline/src/entities/exchange_cancel_event.ts

@@ -48,6 +48,4 @@ export class ExchangeCancelEvent {
     public takerTokenAddress!: string;
     @Column({ nullable: true, type: String, name: 'taker_token_id' })
     public takerTokenId!: string | null;
-
-    // TODO(albrow): Include topics?
 }

packages/pipeline/src/entities/exchange_cancel_up_to_event.ts

@@ -12,7 +12,6 @@ export class ExchangeCancelUpToEvent {
     @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
     public blockNumber!: number;
 
-    // TODO(albrow): Include transaction hash
     @Column({ name: 'raw_data' })
     public rawData!: string;
 
@@ -24,5 +23,4 @@ export class ExchangeCancelUpToEvent {
     public senderAddress!: string;
     @Column({ name: 'order_epoch', type: 'numeric', transformer: bigNumberTransformer })
     public orderEpoch!: BigNumber;
-    // TODO(albrow): Include topics?
 }

packages/pipeline/src/entities/exchange_fill_event.ts

@@ -57,6 +57,4 @@ export class ExchangeFillEvent {
     public takerTokenAddress!: string;
     @Column({ nullable: true, type: String, name: 'taker_token_id' })
     public takerTokenId!: string | null;
-
-    // TODO(albrow): Include topics?
 }

packages/pipeline/src/entities/relayer.ts

@@ -14,9 +14,6 @@ export class Relayer {
     @Column({ name: 'app_url', type: 'varchar', nullable: true })
     public appUrl!: string | null;
 
-    // TODO(albrow): Add exchange contract or protocol version?
-    // TODO(albrow): Add network ids for addresses?
-
     @Column({ name: 'fee_recipient_addresses', type: 'varchar', array: true })
     public feeRecipientAddresses!: string[];
     @Column({ name: 'taker_addresses', type: 'varchar', array: true })

packages/pipeline/src/entities/token_metadata.ts

@@ -1,5 +1,8 @@
+import { BigNumber } from '@0x/utils';
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
+import { bigNumberTransformer } from '../utils/transformers';
+
 @Entity({ name: 'token_metadata', schema: 'raw' })
 export class TokenMetadata {
     @PrimaryColumn({ type: 'varchar', nullable: false })
@@ -8,10 +11,8 @@ export class TokenMetadata {
     @PrimaryColumn({ type: 'varchar', nullable: false })
     public authority!: string;
 
-    // TODO(albrow): Convert decimals field to type BigNumber/numeric because it
-    // comes from a 256-bit integer in a smart contract.
-    @Column({ type: 'integer', nullable: true })
-    public decimals!: number | null;
+    @Column({ type: 'numeric', transformer: bigNumberTransformer, nullable: true })
+    public decimals!: BigNumber | null;
 
     @Column({ type: 'varchar', nullable: true })
     public symbol!: string | null;

packages/pipeline/src/parsers/events/index.ts

@@ -49,8 +49,8 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<Exchang
     exchangeFillEvent.logIndex = eventLog.logIndex as number;
     exchangeFillEvent.rawData = eventLog.data as string;
     exchangeFillEvent.transactionHash = eventLog.transactionHash;
-    exchangeFillEvent.makerAddress = eventLog.args.makerAddress.toString();
-    exchangeFillEvent.takerAddress = eventLog.args.takerAddress.toString();
+    exchangeFillEvent.makerAddress = eventLog.args.makerAddress;
+    exchangeFillEvent.takerAddress = eventLog.args.takerAddress;
     exchangeFillEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress;
     exchangeFillEvent.senderAddress = eventLog.args.senderAddress;
     exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount;
@@ -92,9 +92,8 @@ export function _convertToExchangeCancelEvent(
     exchangeCancelEvent.logIndex = eventLog.logIndex as number;
     exchangeCancelEvent.rawData = eventLog.data as string;
     exchangeCancelEvent.transactionHash = eventLog.transactionHash;
-    exchangeCancelEvent.makerAddress = eventLog.args.makerAddress.toString();
-    exchangeCancelEvent.takerAddress =
-        eventLog.args.takerAddress == null ? null : eventLog.args.takerAddress.toString();
+    exchangeCancelEvent.makerAddress = eventLog.args.makerAddress;
+    exchangeCancelEvent.takerAddress = eventLog.args.takerAddress;
     exchangeCancelEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress;
     exchangeCancelEvent.senderAddress = eventLog.args.senderAddress;
     exchangeCancelEvent.orderHash = eventLog.args.orderHash;
@@ -127,8 +126,8 @@ export function _convertToExchangeCancelUpToEvent(
     exchangeCancelUpToEvent.logIndex = eventLog.logIndex as number;
     exchangeCancelUpToEvent.rawData = eventLog.data as string;
     exchangeCancelUpToEvent.transactionHash = eventLog.transactionHash;
-    exchangeCancelUpToEvent.makerAddress = eventLog.args.makerAddress.toString();
-    exchangeCancelUpToEvent.senderAddress = eventLog.args.senderAddress.toString();
+    exchangeCancelUpToEvent.makerAddress = eventLog.args.makerAddress;
+    exchangeCancelUpToEvent.senderAddress = eventLog.args.senderAddress;
     exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch;
     return exchangeCancelUpToEvent;
 }

packages/pipeline/src/parsers/relayer_registry/index.ts

@@ -18,12 +18,13 @@ function parseRelayer(relayerResp: RelayerResponse, uuid: string): Relayer {
     relayer.name = relayerResp.name;
     relayer.homepageUrl = relayerResp.homepage_url;
     relayer.appUrl = relayerResp.app_url;
-    const mainnet = getMainNetwork(relayerResp);
-    if (mainnet !== undefined) {
-        relayer.sraHttpEndpoint = mainnet.sra_http_endpoint || null;
-        relayer.sraWsEndpoint = mainnet.sra_ws_endpoint || null;
-        relayer.feeRecipientAddresses = R.path(['static_order_fields', 'fee_recipient_addresses'], mainnet) || [];
-        relayer.takerAddresses = R.path(['static_order_fields', 'taker_addresses'], mainnet) || [];
+    const mainNetworkRelayerInfo = getMainNetwork(relayerResp);
+    if (mainNetworkRelayerInfo !== undefined) {
+        relayer.sraHttpEndpoint = mainNetworkRelayerInfo.sra_http_endpoint || null;
+        relayer.sraWsEndpoint = mainNetworkRelayerInfo.sra_ws_endpoint || null;
+        relayer.feeRecipientAddresses =
+            R.path(['static_order_fields', 'fee_recipient_addresses'], mainNetworkRelayerInfo) || [];
+        relayer.takerAddresses = R.path(['static_order_fields', 'taker_addresses'], mainNetworkRelayerInfo) || [];
     } else {
         relayer.feeRecipientAddresses = [];
         relayer.takerAddresses = [];

packages/pipeline/src/parsers/token_metadata/index.ts

@@ -1,7 +1,9 @@
+import { BigNumber } from '@0x/utils';
 import * as R from 'ramda';
 
 import { MetamaskTrustedTokenMeta, ZeroExTrustedTokenMeta } from '../../data_sources/trusted_tokens';
 import { TokenMetadata } from '../../entities';
+import {} from '../../utils';
 
 /**
  * Parses Metamask's trusted tokens list.
@@ -24,7 +26,7 @@ function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: stri
     const trustedToken = new TokenMetadata();
 
     trustedToken.address = address;
-    trustedToken.decimals = resp.decimals;
+    trustedToken.decimals = new BigNumber(resp.decimals);
     trustedToken.symbol = resp.symbol;
     trustedToken.name = resp.name;
     trustedToken.authority = 'metamask';
@@ -36,7 +38,7 @@ function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TokenMetadata {
     const trustedToken = new TokenMetadata();
 
     trustedToken.address = resp.address;
-    trustedToken.decimals = resp.decimals;
+    trustedToken.decimals = new BigNumber(resp.decimals);
     trustedToken.symbol = resp.symbol;
     trustedToken.name = resp.name;
     trustedToken.authority = '0x';

packages/pipeline/src/parsers/web3/index.ts

@@ -42,7 +42,6 @@ export function parseTransaction(rawTransaction: EthTransaction): Transaction {
     tx.blockNumber = rawTransaction.blockNumber;
 
     tx.gasUsed = rawTransaction.gas;
-    // TODO(albrow) figure out bignum solution.
     tx.gasPrice = rawTransaction.gasPrice.toNumber();
 
     return tx;

packages/pipeline/src/scripts/pull_missing_blocks.ts

@@ -9,7 +9,7 @@ import { Web3Source } from '../data_sources/web3';
 import { Block } from '../entities';
 import * as ormConfig from '../ormconfig';
 import { parseBlock } from '../parsers/web3';
-import { handleError } from '../utils';
+import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils';
 
 // Number of blocks to save at once.
 const BATCH_SAVE_SIZE = 1000;
@@ -18,16 +18,13 @@ const MAX_CONCURRENCY = 10;
 // Maximum number of blocks to query for at once. This is also the maximum
 // number of blocks we will hold in memory prior to being saved to the database.
 const MAX_BLOCKS_PER_QUERY = 1000;
-// Block number when the Exchange contract was deployed to mainnet.
-// TODO(albrow): De-dupe this constant.
-const EXCHANGE_START_BLOCK = 6271590;
 
 let connection: Connection;
 
 (async () => {
     connection = await createConnection(ormConfig as ConnectionOptions);
     const provider = web3Factory.getRpcProvider({
-        rpcUrl: `https://mainnet.infura.io/${process.env.INFURA_API_KEY}`,
+        rpcUrl: `${INFURA_ROOT_URL}/${process.env.INFURA_API_KEY}`,
     });
     const web3Source = new Web3Source(provider);
     await getAllMissingBlocks(web3Source);

packages/pipeline/src/scripts/pull_missing_events.ts

@@ -8,9 +8,8 @@ import { ExchangeEventsSource } from '../data_sources/contract-wrappers/exchange
 import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeEvent, ExchangeFillEvent } from '../entities';
 import * as ormConfig from '../ormconfig';
 import { parseExchangeCancelEvents, parseExchangeCancelUpToEvents, parseExchangeFillEvents } from '../parsers/events';
-import { handleError } from '../utils';
+import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils';
 
-const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
 const START_BLOCK_OFFSET = 100; // Number of blocks before the last known block to consider when updating fill events.
 const BATCH_SAVE_SIZE = 1000; // Number of events to save at once.
 
@@ -19,7 +18,7 @@ let connection: Connection;
 (async () => {
     connection = await createConnection(ormConfig as ConnectionOptions);
     const provider = web3Factory.getRpcProvider({
-        rpcUrl: 'https://mainnet.infura.io',
+        rpcUrl: INFURA_ROOT_URL,
     });
     const eventsSource = new ExchangeEventsSource(provider, 1);
     await getFillEventsAsync(eventsSource);

packages/pipeline/src/scripts/pull_radar_relay_orders.ts

@@ -16,11 +16,11 @@ let connection: Connection;
 
 (async () => {
     connection = await createConnection(ormConfig as ConnectionOptions);
-    await getOrderbook();
+    await getOrderbookAsync();
     process.exit(0);
 })().catch(handleError);
 
-async function getOrderbook(): Promise<void> {
+async function getOrderbookAsync(): Promise<void> {
     console.log('Getting all orders...');
     const connectClient = new HttpClient(RADAR_RELAY_URL);
     const rawOrders = await connectClient.getOrdersAsync({
@@ -29,17 +29,22 @@ async function getOrderbook(): Promise<void> {
     console.log(`Got ${rawOrders.records.length} orders.`);
     console.log('Parsing orders...');
     // Parse the sra orders, then add source url to each.
-    const orders = R.pipe(parseSraOrders, R.map(setSourceUrl(RADAR_RELAY_URL)))(rawOrders);
+    const orders = R.pipe(
+        parseSraOrders,
+        R.map(setSourceUrl(RADAR_RELAY_URL)),
+    )(rawOrders);
     // Save all the orders and update the observed time stamps in a single
     // transaction.
     console.log('Saving orders and updating timestamps...');
-    await connection.transaction(async (manager: EntityManager): Promise<void> => {
-        for (const order of orders) {
-            await manager.save(SraOrder, order);
-            const observedTimestamp = createObservedTimestampForOrder(order);
-            await manager.save(observedTimestamp);
-        }
-    });
+    await connection.transaction(
+        async (manager: EntityManager): Promise<void> => {
+            for (const order of orders) {
+                await manager.save(SraOrder, order);
+                const observedTimestamp = createObservedTimestampForOrder(order);
+                await manager.save(observedTimestamp);
+            }
+        },
+    );
 }
 
 const sourceUrlProp = R.lensProp('sourceUrl');
@@ -48,6 +53,8 @@ const sourceUrlProp = R.lensProp('sourceUrl');
  * Sets the source url for a single order. Returns a new order instead of
  * mutating the given one.
  */
-const setSourceUrl = R.curry((sourceURL: string, order: SraOrder): SraOrder => {
-    return R.set(sourceUrlProp, sourceURL, order);
-});
+const setSourceUrl = R.curry(
+    (sourceURL: string, order: SraOrder): SraOrder => {
+        return R.set(sourceUrlProp, sourceURL, order);
+    },
+);

packages/pipeline/src/scripts/pull_trusted_tokens.ts

@@ -8,7 +8,7 @@ import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers
 import { handleError } from '../utils';
 
 const METAMASK_TRUSTED_TOKENS_URL =
-    'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/master/contract-map.json';
+    'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/d45916c533116510cc8e9e048a8b5fc3732a6b6d/contract-map.json';
 
 const ZEROEX_TRUSTED_TOKENS_URL = 'https://website-api.0xproject.com/tokens';
 
@@ -22,7 +22,7 @@ let connection: Connection;
 })().catch(handleError);
 
 async function getMetamaskTrustedTokens(): Promise<void> {
-    // tslint:disable-next-line
+    // tslint:disable-next-line:no-console
     console.log('Getting latest metamask trusted tokens list ...');
     const trustedTokensRepository = connection.getRepository(TokenMetadata);
     const trustedTokensSource = new TrustedTokenSource<Map<string, MetamaskTrustedTokenMeta>>(
@@ -30,23 +30,23 @@ async function getMetamaskTrustedTokens(): Promise<void> {
     );
     const resp = await trustedTokensSource.getTrustedTokenMetaAsync();
     const trustedTokens = parseMetamaskTrustedTokens(resp);
-    // tslint:disable-next-line
+    // tslint:disable-next-line:no-console
     console.log('Saving metamask trusted tokens list');
     await trustedTokensRepository.save(trustedTokens);
-    // tslint:disable-next-line
+    // tslint:disable-next-line:no-console
     console.log('Done saving metamask trusted tokens.');
 }
 
 async function getZeroExTrustedTokens(): Promise<void> {
-    // tslint:disable-next-line
+    // tslint:disable-next-line:no-console
     console.log('Getting latest 0x trusted tokens list ...');
     const trustedTokensRepository = connection.getRepository(TokenMetadata);
     const trustedTokensSource = new TrustedTokenSource<ZeroExTrustedTokenMeta[]>(ZEROEX_TRUSTED_TOKENS_URL);
     const resp = await trustedTokensSource.getTrustedTokenMetaAsync();
     const trustedTokens = parseZeroExTrustedTokens(resp);
-    // tslint:disable-next-line
+    // tslint:disable-next-line:no-console
     console.log('Saving metamask trusted tokens list');
     await trustedTokensRepository.save(trustedTokens);
-    // tslint:disable-next-line
+    // tslint:disable-next-line:no-console
     console.log('Done saving metamask trusted tokens.');
 }

packages/pipeline/src/utils/constants.ts

@@ -0,0 +1,3 @@
+// Block number when the Exchange contract was deployed to mainnet.
+export const EXCHANGE_START_BLOCK = 6271590;
+export const INFURA_ROOT_URL = 'https://mainnet.infura.io';

packages/pipeline/src/utils/index.ts

@@ -1,5 +1,6 @@
 import { BigNumber } from '@0x/utils';
 export * from './transformers';
+export * from './constants';
 
 /**
  * If the given BigNumber is not null, returns the string representation of that

packages/pipeline/src/utils/transformers/big_number.ts

@@ -3,13 +3,13 @@ import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer';
 
 export class BigNumberTransformer implements ValueTransformer {
     // tslint:disable-next-line:prefer-function-over-method
-    public to(value: BigNumber): string {
-        return value.toString();
+    public to(value: BigNumber | null): string | null {
+        return value === null ? null : value.toString();
     }
 
     // tslint:disable-next-line:prefer-function-over-method
-    public from(value: string): BigNumber {
-        return new BigNumber(value);
+    public from(value: string | null): BigNumber | null {
+        return value === null ? null : new BigNumber(value);
     }
 }
 

packages/pipeline/test/entities/token_metadata_test.ts

@@ -1,3 +1,4 @@
+import { BigNumber } from '@0x/utils';
 import 'mocha';
 import 'reflect-metadata';
 
@@ -9,15 +10,15 @@ import { testSaveAndFindEntityAsync } from './util';
 
 chaiSetup.configure();
 
-const metadataWithoutNullFields = {
+const metadataWithoutNullFields: TokenMetadata = {
     address: '0xe41d2489571d322189246dafa5ebde1f4699f498',
     authority: 'https://website-api.0xproject.com/tokens',
-    decimals: 18,
+    decimals: new BigNumber(18),
     symbol: 'ZRX',
     name: '0x',
 };
 
-const metadataWithNullFields = {
+const metadataWithNullFields: TokenMetadata = {
     address: '0xe41d2489571d322189246dafa5ebde1f4699f499',
     authority: 'https://website-api.0xproject.com/tokens',
     decimals: null,