From 7ad4cb00780329115e831b304aebfae1792f7c4c Mon Sep 17 00:00:00 2001 From: askeluv Date: Fri, 8 Feb 2019 18:54:38 +0800 Subject: [PATCH 01/37] [WIP] Slippage calculations from Ethereum DEX Prices Service (EDPS) --- .../src/data_sources/slippage/index.ts | 59 +++++++++++++++++++ packages/pipeline/src/entities/index.ts | 1 + packages/pipeline/src/entities/slippage.ts | 15 +++++ packages/pipeline/src/ormconfig.ts | 2 + .../pipeline/src/parsers/slippage/index.ts | 35 +++++++++++ .../pipeline/src/scripts/pull_slippage.ts | 38 ++++++++++++ packages/pipeline/test/entities/slippage.ts | 32 ++++++++++ 7 files changed, 182 insertions(+) create mode 100644 packages/pipeline/src/data_sources/slippage/index.ts create mode 100644 packages/pipeline/src/entities/slippage.ts create mode 100644 packages/pipeline/src/parsers/slippage/index.ts create mode 100644 packages/pipeline/src/scripts/pull_slippage.ts create mode 100644 packages/pipeline/test/entities/slippage.ts diff --git a/packages/pipeline/src/data_sources/slippage/index.ts b/packages/pipeline/src/data_sources/slippage/index.ts new file mode 100644 index 0000000000..cf746e4ba0 --- /dev/null +++ b/packages/pipeline/src/data_sources/slippage/index.ts @@ -0,0 +1,59 @@ +import { fetchAsync, logUtils } from '@0x/utils'; + +const EDPS_BASE_URL = 'https://ethereum-dex-prices-service.production.airswap.io'; +const PRICE_BASE_URL = 'https://min-api.cryptocompare.com/data/price?tsyms=USD' + +export type EdpsResponse = EdpsWrapper[] + +export interface EdpsWrapper { + [key: string]: EdpsExchange +} + +export interface EdpsExchange { + exchangeName: string, + totalPrice: number, + tokenAmount: number, + tokenSymbol: string, + avgPrice: number, + timestamp: number, + error: string +} + +export interface PriceResponse { + USD: number; +} + +// tslint:disable:prefer-function-over-method +// ^ Keep consistency with other sources and help logical organization +export class EdpsSource { + /** + * Call Ethereum DEX Price Service API. + */ + public async getEdpsAsync(direction: string, symbol: string, amount: number): Promise> { + logUtils.log('Getting EDPS response'); + const edpsUrl = `${EDPS_BASE_URL}/${direction}?symbol=${symbol}&amount=${amount}`; + const resp = await fetchAsync(edpsUrl); + const respJson: EdpsResponse = await resp.json(); + const allExchanges = new Map(); + for (let entry of respJson) { + for (let key in entry) { + allExchanges.set(key, entry[key]); + } + } + logUtils.log(`Got ${allExchanges.size} exchanges.`); + return allExchanges; + } +} + +export class PriceSource { + /** + * Call CryptoCompare Price API to get USD price of token. + */ + public async getUsdPriceAsync(symbol: string): Promise { + logUtils.log(`Fetching USD price for ${symbol}`); + const priceUrl = `${PRICE_BASE_URL}&fsym=${symbol}` + const resp = await fetchAsync(priceUrl); + const respJson: PriceResponse = await resp.json(); + return respJson.USD; + } +} \ No newline at end of file diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts index 27c153c079..c1db5b9667 100644 --- a/packages/pipeline/src/entities/index.ts +++ b/packages/pipeline/src/entities/index.ts @@ -9,6 +9,7 @@ export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event'; export { ExchangeFillEvent } from './exchange_fill_event'; export { OHLCVExternal } from './ohlcv_external'; export { Relayer } from './relayer'; +export { SlippageRecord } from './slippage'; export { SraOrder } from './sra_order'; export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp'; export { TokenMetadata } from './token_metadata'; diff --git a/packages/pipeline/src/entities/slippage.ts b/packages/pipeline/src/entities/slippage.ts new file mode 100644 index 0000000000..a430653802 --- /dev/null +++ b/packages/pipeline/src/entities/slippage.ts @@ -0,0 +1,15 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +@Entity({ name: 'slippage_records', schema: 'raw' }) +export class SlippageRecord { + @PrimaryColumn({ name: 'time', type: 'number'}) + public time!: number; + @PrimaryColumn({ name: 'symbol' }) + public symbol!: string; + @PrimaryColumn({ name: 'exchange' }) + public exchange!: string; + @PrimaryColumn({ name: 'usdAmount', type: 'number' }) + public usdAmount!: number; + @Column({ name: 'slippage', type: 'number' }) + public slippage!: number; +} diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 2700714cdd..815b859655 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -14,6 +14,7 @@ import { ExchangeFillEvent, OHLCVExternal, Relayer, + SlippageRecord, SraOrder, SraOrdersObservedTimeStamp, TokenMetadata, @@ -35,6 +36,7 @@ const entities = [ ERC20ApprovalEvent, OHLCVExternal, Relayer, + SlippageRecord, SraOrder, SraOrdersObservedTimeStamp, TokenMetadata, diff --git a/packages/pipeline/src/parsers/slippage/index.ts b/packages/pipeline/src/parsers/slippage/index.ts new file mode 100644 index 0000000000..24c7b9e2f5 --- /dev/null +++ b/packages/pipeline/src/parsers/slippage/index.ts @@ -0,0 +1,35 @@ +import { BigNumber } from '@0x/utils'; +import * as R from 'ramda'; + +import { EdpsExchange } from '../../data_sources/slippage'; +import { SlippageRecord } from '../../entities'; +import { symbol } from 'prop-types'; + +/** + * Calculates slippage and returns SlippageRecord entity. + * + * @param usdAmount + * @param exchange + * @param buyEdps + * @param sellEdps + */ + + export function calculateSlippage(usdAmount: number, exchange: string, + buyEdps: Map, sellEdps: Map) { + const b = buyEdps.get(exchange); + const s = sellEdps.get(exchange); + if (b && s && b.avgPrice && s.avgPrice) { + var slippage = (b.avgPrice - s.avgPrice) / b.avgPrice; + const observedTimestamp = Date.now(); + const slippageRecord = new SlippageRecord(); + slippageRecord.time = observedTimestamp; + slippageRecord.symbol = b.tokenSymbol; + slippageRecord.exchange = exchange; + slippageRecord.usdAmount = usdAmount; + slippageRecord.slippage = slippage; + return slippageRecord; + } + else { + return null; + } + } diff --git a/packages/pipeline/src/scripts/pull_slippage.ts b/packages/pipeline/src/scripts/pull_slippage.ts new file mode 100644 index 0000000000..9faa69376d --- /dev/null +++ b/packages/pipeline/src/scripts/pull_slippage.ts @@ -0,0 +1,38 @@ +import * as R from 'ramda'; +import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm'; +import { logUtils } from '@0x/utils'; +import { EdpsExchange, EdpsSource, PriceResponse, PriceSource } from '../data_sources/slippage'; +import { handleError } from '../utils'; +import { string, number } from 'prop-types'; +import { calculateSlippage } from '../parsers/slippage'; +import { SlippageRecord } from '../entities'; +import * as ormConfig from '../ormconfig'; + +// Number of orders to save at once. +const BATCH_SAVE_SIZE = 1000; + +// USD amounts for slippage depths +const USD_AMOUNTS = [10, 100, 1000]; +const TOKENS = ['ZRX', 'MKR', 'DAI', 'KNC', 'BNB']; // TODO: fetch from database + +(async () => { + const priceSource = new PriceSource(); + const edpsSource = new EdpsSource(); + const resultsPerAmount = await TOKENS.map(async (symbol) => { + const usdPrice = await priceSource.getUsdPriceAsync(symbol); + USD_AMOUNTS.map(async (usdAmount) => { + const amount = usdAmount / usdPrice; + console.log(amount); + const buyEdps = await edpsSource.getEdpsAsync('buy', symbol, amount); + const sellEdps = await edpsSource.getEdpsAsync('sell', symbol, amount); + + for(let exchange of buyEdps.keys()) { + const slippageRecord = await calculateSlippage(usdAmount, exchange, buyEdps, sellEdps) + if (slippageRecord) + console.log(slippageRecord); + } + + } + )}); + //process.exit(0); +})().catch(handleError); \ No newline at end of file diff --git a/packages/pipeline/test/entities/slippage.ts b/packages/pipeline/test/entities/slippage.ts new file mode 100644 index 0000000000..ea408f8bec --- /dev/null +++ b/packages/pipeline/test/entities/slippage.ts @@ -0,0 +1,32 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; +import * as R from 'ramda'; +import 'reflect-metadata'; + +import { SlippageRecord } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const slippageRecord = { + time: 1234, + symbol: 'ZRX', + exchange: 'Paradex', + usdAmount: 10, + slippage: 0.01 +}; + +// tslint:disable:custom-no-magic-numbers +describe('Slippage entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const slippageRecords = [slippageRecord]; + const slippageRepository = connection.getRepository(SlippageRecord); + for (const record of slippageRecords) { + await testSaveAndFindEntityAsync(slippageRepository, record); + } + }); +}); From c0a2f429b6f34d75af0393453baca0b579531c8c Mon Sep 17 00:00:00 2001 From: askeluv Date: Tue, 12 Feb 2019 12:30:10 +0800 Subject: [PATCH 02/37] Added saving to database --- .../1549856835629-CreateSlippageTable.ts | 28 +++++++++++ .../src/data_sources/slippage/index.ts | 3 -- packages/pipeline/src/entities/index.ts | 2 +- packages/pipeline/src/entities/slippage.ts | 21 ++++++--- packages/pipeline/src/ormconfig.ts | 4 +- .../pipeline/src/parsers/slippage/index.ts | 27 ++++++----- .../pipeline/src/scripts/pull_slippage.ts | 46 ++++++++++++------- packages/pipeline/test/entities/slippage.ts | 19 ++++---- 8 files changed, 98 insertions(+), 52 deletions(-) create mode 100644 packages/pipeline/migrations/1549856835629-CreateSlippageTable.ts diff --git a/packages/pipeline/migrations/1549856835629-CreateSlippageTable.ts b/packages/pipeline/migrations/1549856835629-CreateSlippageTable.ts new file mode 100644 index 0000000000..d9d6ff4f61 --- /dev/null +++ b/packages/pipeline/migrations/1549856835629-CreateSlippageTable.ts @@ -0,0 +1,28 @@ +import {MigrationInterface, QueryRunner, Table} from "typeorm"; + +const slippage = new Table({ + name: 'raw.slippage', + columns: [ + { name: 'observed_timestamp', type: 'bigint', isPrimary: true }, + { name: 'symbol', type: 'varchar', isPrimary: true }, + { name: 'exchange', type: 'varchar', isPrimary: true }, + { name: 'usd_amount', type: 'numeric', isPrimary: true }, + + { name: 'token_amount', type: 'numeric', isNullable: false }, + { name: 'avg_price_in_eth_buy', type: 'numeric', isNullable: true }, + { name: 'avg_price_in_eth_sell', type: 'numeric', isNullable: true }, + { name: 'slippage', type: 'numeric', isNullable: true }, + ], +}); + +export class CreateSlippageTable1549856835629 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable(slippage); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable(slippage); + } + +} diff --git a/packages/pipeline/src/data_sources/slippage/index.ts b/packages/pipeline/src/data_sources/slippage/index.ts index cf746e4ba0..57bb2cc4a2 100644 --- a/packages/pipeline/src/data_sources/slippage/index.ts +++ b/packages/pipeline/src/data_sources/slippage/index.ts @@ -30,7 +30,6 @@ export class EdpsSource { * Call Ethereum DEX Price Service API. */ public async getEdpsAsync(direction: string, symbol: string, amount: number): Promise> { - logUtils.log('Getting EDPS response'); const edpsUrl = `${EDPS_BASE_URL}/${direction}?symbol=${symbol}&amount=${amount}`; const resp = await fetchAsync(edpsUrl); const respJson: EdpsResponse = await resp.json(); @@ -40,7 +39,6 @@ export class EdpsSource { allExchanges.set(key, entry[key]); } } - logUtils.log(`Got ${allExchanges.size} exchanges.`); return allExchanges; } } @@ -50,7 +48,6 @@ export class PriceSource { * Call CryptoCompare Price API to get USD price of token. */ public async getUsdPriceAsync(symbol: string): Promise { - logUtils.log(`Fetching USD price for ${symbol}`); const priceUrl = `${PRICE_BASE_URL}&fsym=${symbol}` const resp = await fetchAsync(priceUrl); const respJson: PriceResponse = await resp.json(); diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts index c1db5b9667..9679c4005f 100644 --- a/packages/pipeline/src/entities/index.ts +++ b/packages/pipeline/src/entities/index.ts @@ -9,7 +9,7 @@ export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event'; export { ExchangeFillEvent } from './exchange_fill_event'; export { OHLCVExternal } from './ohlcv_external'; export { Relayer } from './relayer'; -export { SlippageRecord } from './slippage'; +export { Slippage } from './slippage'; export { SraOrder } from './sra_order'; export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp'; export { TokenMetadata } from './token_metadata'; diff --git a/packages/pipeline/src/entities/slippage.ts b/packages/pipeline/src/entities/slippage.ts index a430653802..6390e82fba 100644 --- a/packages/pipeline/src/entities/slippage.ts +++ b/packages/pipeline/src/entities/slippage.ts @@ -1,15 +1,22 @@ import { Column, Entity, PrimaryColumn } from 'typeorm'; +import { numberToBigIntTransformer } from '../utils'; -@Entity({ name: 'slippage_records', schema: 'raw' }) -export class SlippageRecord { - @PrimaryColumn({ name: 'time', type: 'number'}) - public time!: number; +@Entity({ name: 'slippage', schema: 'raw' }) +export class Slippage { + @PrimaryColumn({ name: 'observed_timestamp', type: 'bigint', transformer: numberToBigIntTransformer}) + public observedTimestamp!: number; @PrimaryColumn({ name: 'symbol' }) public symbol!: string; @PrimaryColumn({ name: 'exchange' }) public exchange!: string; - @PrimaryColumn({ name: 'usdAmount', type: 'number' }) + @PrimaryColumn({ name: 'usd_amount', type: 'numeric' }) public usdAmount!: number; - @Column({ name: 'slippage', type: 'number' }) - public slippage!: number; + @PrimaryColumn({ name: 'token_amount', type: 'numeric' }) + public tokenAmount!: number; + @PrimaryColumn({ name: 'avg_price_in_eth_sell', type: 'numeric' }) + public avgPriceInEthSell?: number; + @PrimaryColumn({ name: 'avg_price_in_eth_buy', type: 'numeric' }) + public avgPriceInEthBuy?: number; + @Column({ name: 'slippage', type: 'numeric' }) + public slippage?: number; } diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 815b859655..92abbf4048 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -14,7 +14,7 @@ import { ExchangeFillEvent, OHLCVExternal, Relayer, - SlippageRecord, + Slippage, SraOrder, SraOrdersObservedTimeStamp, TokenMetadata, @@ -36,7 +36,7 @@ const entities = [ ERC20ApprovalEvent, OHLCVExternal, Relayer, - SlippageRecord, + Slippage, SraOrder, SraOrdersObservedTimeStamp, TokenMetadata, diff --git a/packages/pipeline/src/parsers/slippage/index.ts b/packages/pipeline/src/parsers/slippage/index.ts index 24c7b9e2f5..84b9b23410 100644 --- a/packages/pipeline/src/parsers/slippage/index.ts +++ b/packages/pipeline/src/parsers/slippage/index.ts @@ -2,11 +2,11 @@ import { BigNumber } from '@0x/utils'; import * as R from 'ramda'; import { EdpsExchange } from '../../data_sources/slippage'; -import { SlippageRecord } from '../../entities'; +import { Slippage } from '../../entities'; import { symbol } from 'prop-types'; /** - * Calculates slippage and returns SlippageRecord entity. + * Calculates slippage and returns Slippage entity. * * @param usdAmount * @param exchange @@ -18,18 +18,17 @@ import { symbol } from 'prop-types'; buyEdps: Map, sellEdps: Map) { const b = buyEdps.get(exchange); const s = sellEdps.get(exchange); + const slippage = new Slippage(); if (b && s && b.avgPrice && s.avgPrice) { - var slippage = (b.avgPrice - s.avgPrice) / b.avgPrice; - const observedTimestamp = Date.now(); - const slippageRecord = new SlippageRecord(); - slippageRecord.time = observedTimestamp; - slippageRecord.symbol = b.tokenSymbol; - slippageRecord.exchange = exchange; - slippageRecord.usdAmount = usdAmount; - slippageRecord.slippage = slippage; - return slippageRecord; - } - else { - return null; + slippage.observedTimestamp = b.timestamp; + slippage.symbol = b.tokenSymbol; + slippage.exchange = exchange; + slippage.usdAmount = usdAmount; + slippage.tokenAmount = Number(b.tokenAmount); // API returns a string + slippage.avgPriceInEthBuy = b.avgPrice; + slippage.avgPriceInEthSell = s.avgPrice; + slippage.slippage = (b.avgPrice - s.avgPrice) / b.avgPrice; + } + return slippage; } diff --git a/packages/pipeline/src/scripts/pull_slippage.ts b/packages/pipeline/src/scripts/pull_slippage.ts index 9faa69376d..9411e51389 100644 --- a/packages/pipeline/src/scripts/pull_slippage.ts +++ b/packages/pipeline/src/scripts/pull_slippage.ts @@ -1,38 +1,50 @@ import * as R from 'ramda'; -import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm'; +import { Connection, ConnectionOptions, createConnection, Repository, PromiseUtils, AdvancedConsoleLogger } from 'typeorm'; import { logUtils } from '@0x/utils'; import { EdpsExchange, EdpsSource, PriceResponse, PriceSource } from '../data_sources/slippage'; import { handleError } from '../utils'; import { string, number } from 'prop-types'; import { calculateSlippage } from '../parsers/slippage'; -import { SlippageRecord } from '../entities'; +import { Slippage } from '../entities'; import * as ormConfig from '../ormconfig'; // Number of orders to save at once. const BATCH_SAVE_SIZE = 1000; // USD amounts for slippage depths -const USD_AMOUNTS = [10, 100, 1000]; -const TOKENS = ['ZRX', 'MKR', 'DAI', 'KNC', 'BNB']; // TODO: fetch from database +const USD_AMOUNTS = [10, 100, 1000, 10000]; + +// TODO: fetch from database +const TOKENS = ['BAT', 'DAI', 'FUN', 'MANA', 'OMG', 'REP', 'TUSD', 'ZRX', 'MKR', 'BNB', 'USDC']; + +let connection: Connection; (async () => { + connection = await createConnection(ormConfig as ConnectionOptions); const priceSource = new PriceSource(); const edpsSource = new EdpsSource(); - const resultsPerAmount = await TOKENS.map(async (symbol) => { + + logUtils.log('Fetching slippage records'); + let nestedSlippages: Slippage[][][] = await Promise.all(await TOKENS.map(async (symbol) => { const usdPrice = await priceSource.getUsdPriceAsync(symbol); - USD_AMOUNTS.map(async (usdAmount) => { + return Promise.all(USD_AMOUNTS.map(async (usdAmount) => { const amount = usdAmount / usdPrice; - console.log(amount); const buyEdps = await edpsSource.getEdpsAsync('buy', symbol, amount); const sellEdps = await edpsSource.getEdpsAsync('sell', symbol, amount); - - for(let exchange of buyEdps.keys()) { - const slippageRecord = await calculateSlippage(usdAmount, exchange, buyEdps, sellEdps) - if (slippageRecord) - console.log(slippageRecord); - } - - } - )}); - //process.exit(0); + const slippages = Array.from(buyEdps.keys()).map((exchange) => { + const slippage: Slippage = calculateSlippage(usdAmount, exchange, buyEdps, sellEdps); + return slippage; + }); + return slippages; + })); + })); + let slippagesWithEmptyRecords = await nestedSlippages + .reduce((acc, val) => acc.concat(val)) + .reduce((acc, val) => acc.concat(val)); + let slippages = slippagesWithEmptyRecords.filter((slippage) => slippage.observedTimestamp) + const SlippageRepository = connection.getRepository(Slippage); + logUtils.log(`Saving ${slippages.length} records to database`); + await SlippageRepository.save(slippages, { chunk: Math.ceil(slippages.length / BATCH_SAVE_SIZE) }); + logUtils.log("Done"); + process.exit(0); })().catch(handleError); \ No newline at end of file diff --git a/packages/pipeline/test/entities/slippage.ts b/packages/pipeline/test/entities/slippage.ts index ea408f8bec..a342f0ef14 100644 --- a/packages/pipeline/test/entities/slippage.ts +++ b/packages/pipeline/test/entities/slippage.ts @@ -3,7 +3,7 @@ import 'mocha'; import * as R from 'ramda'; import 'reflect-metadata'; -import { SlippageRecord } from '../../src/entities'; +import { Slippage } from '../../src/entities'; import { createDbConnectionOnceAsync } from '../db_setup'; import { chaiSetup } from '../utils/chai_setup'; @@ -11,11 +11,14 @@ import { testSaveAndFindEntityAsync } from './util'; chaiSetup.configure(); -const slippageRecord = { - time: 1234, +const slippage = { + observedTimestamp: 1549587475793, symbol: 'ZRX', - exchange: 'Paradex', + exchange: 'Radar Relay', usdAmount: 10, + tokenAmount: 25, + avgPriceInEthBuy: 0.0022, + avgPriceInEthSell: 0.002, slippage: 0.01 }; @@ -23,10 +26,10 @@ const slippageRecord = { describe('Slippage entity', () => { it('save/find', async () => { const connection = await createDbConnectionOnceAsync(); - const slippageRecords = [slippageRecord]; - const slippageRepository = connection.getRepository(SlippageRecord); - for (const record of slippageRecords) { - await testSaveAndFindEntityAsync(slippageRepository, record); + const slippages = [slippage]; + const slippageRepository = connection.getRepository(Slippage); + for (const slippage of slippages) { + await testSaveAndFindEntityAsync(slippageRepository, slippage); } }); }); From d7825dd7db08bf8bf1ddf1e93388f313372eb525 Mon Sep 17 00:00:00 2001 From: askeluv Date: Tue, 12 Feb 2019 16:06:53 +0800 Subject: [PATCH 03/37] Added test for entity --- packages/pipeline/src/entities/slippage.ts | 23 ++++++++++--------- .../pipeline/src/parsers/slippage/index.ts | 10 ++++---- .../{slippage.ts => slippage_test.ts} | 10 ++++---- 3 files changed, 22 insertions(+), 21 deletions(-) rename packages/pipeline/test/entities/{slippage.ts => slippage_test.ts} (81%) diff --git a/packages/pipeline/src/entities/slippage.ts b/packages/pipeline/src/entities/slippage.ts index 6390e82fba..d046f382b5 100644 --- a/packages/pipeline/src/entities/slippage.ts +++ b/packages/pipeline/src/entities/slippage.ts @@ -1,5 +1,6 @@ +import { BigNumber } from '@0x/utils'; import { Column, Entity, PrimaryColumn } from 'typeorm'; -import { numberToBigIntTransformer } from '../utils'; +import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; @Entity({ name: 'slippage', schema: 'raw' }) export class Slippage { @@ -9,14 +10,14 @@ export class Slippage { public symbol!: string; @PrimaryColumn({ name: 'exchange' }) public exchange!: string; - @PrimaryColumn({ name: 'usd_amount', type: 'numeric' }) - public usdAmount!: number; - @PrimaryColumn({ name: 'token_amount', type: 'numeric' }) - public tokenAmount!: number; - @PrimaryColumn({ name: 'avg_price_in_eth_sell', type: 'numeric' }) - public avgPriceInEthSell?: number; - @PrimaryColumn({ name: 'avg_price_in_eth_buy', type: 'numeric' }) - public avgPriceInEthBuy?: number; - @Column({ name: 'slippage', type: 'numeric' }) - public slippage?: number; + @PrimaryColumn({ name: 'usd_amount', type: 'numeric', transformer: bigNumberTransformer }) + public usdAmount!: BigNumber; + @PrimaryColumn({ name: 'token_amount', type: 'numeric', transformer: bigNumberTransformer }) + public tokenAmount!: BigNumber; + @PrimaryColumn({ name: 'avg_price_in_eth_sell', type: 'numeric', transformer: bigNumberTransformer }) + public avgPriceInEthSell?: BigNumber; + @PrimaryColumn({ name: 'avg_price_in_eth_buy', type: 'numeric', transformer: bigNumberTransformer }) + public avgPriceInEthBuy?: BigNumber; + @Column({ name: 'slippage', type: 'numeric', transformer: bigNumberTransformer }) + public slippage?: BigNumber; } diff --git a/packages/pipeline/src/parsers/slippage/index.ts b/packages/pipeline/src/parsers/slippage/index.ts index 84b9b23410..e38fb4c6a7 100644 --- a/packages/pipeline/src/parsers/slippage/index.ts +++ b/packages/pipeline/src/parsers/slippage/index.ts @@ -23,11 +23,11 @@ import { symbol } from 'prop-types'; slippage.observedTimestamp = b.timestamp; slippage.symbol = b.tokenSymbol; slippage.exchange = exchange; - slippage.usdAmount = usdAmount; - slippage.tokenAmount = Number(b.tokenAmount); // API returns a string - slippage.avgPriceInEthBuy = b.avgPrice; - slippage.avgPriceInEthSell = s.avgPrice; - slippage.slippage = (b.avgPrice - s.avgPrice) / b.avgPrice; + slippage.usdAmount = new BigNumber(usdAmount); + slippage.tokenAmount = new BigNumber(Number(b.tokenAmount)); // API returns a string + slippage.avgPriceInEthBuy = new BigNumber(b.avgPrice); + slippage.avgPriceInEthSell = new BigNumber(s.avgPrice); + slippage.slippage = new BigNumber((b.avgPrice - s.avgPrice) / b.avgPrice); } return slippage; diff --git a/packages/pipeline/test/entities/slippage.ts b/packages/pipeline/test/entities/slippage_test.ts similarity index 81% rename from packages/pipeline/test/entities/slippage.ts rename to packages/pipeline/test/entities/slippage_test.ts index a342f0ef14..5f95e28c5e 100644 --- a/packages/pipeline/test/entities/slippage.ts +++ b/packages/pipeline/test/entities/slippage_test.ts @@ -15,11 +15,11 @@ const slippage = { observedTimestamp: 1549587475793, symbol: 'ZRX', exchange: 'Radar Relay', - usdAmount: 10, - tokenAmount: 25, - avgPriceInEthBuy: 0.0022, - avgPriceInEthSell: 0.002, - slippage: 0.01 + usdAmount: new BigNumber(10), + tokenAmount: new BigNumber(25), + avgPriceInEthBuy: new BigNumber(0.0022), + avgPriceInEthSell: new BigNumber(0.002), + slippage: new BigNumber(0.01) }; // tslint:disable:custom-no-magic-numbers From 82dffe9d0e918448a98cf07f55890d3558983100 Mon Sep 17 00:00:00 2001 From: askeluv Date: Tue, 12 Feb 2019 19:34:05 +0800 Subject: [PATCH 04/37] Added remaining tests + fixed linting issues --- .../src/data_sources/slippage/index.ts | 35 +++++----- packages/pipeline/src/entities/slippage.ts | 4 +- .../pipeline/src/parsers/slippage/index.ts | 48 +++++++------- .../pipeline/src/scripts/pull_slippage.ts | 33 +++++----- .../pipeline/test/entities/slippage_test.ts | 7 +- .../test/parsers/slippage/index_test.ts | 64 +++++++++++++++++++ 6 files changed, 127 insertions(+), 64 deletions(-) create mode 100644 packages/pipeline/test/parsers/slippage/index_test.ts diff --git a/packages/pipeline/src/data_sources/slippage/index.ts b/packages/pipeline/src/data_sources/slippage/index.ts index 57bb2cc4a2..65bb8fe046 100644 --- a/packages/pipeline/src/data_sources/slippage/index.ts +++ b/packages/pipeline/src/data_sources/slippage/index.ts @@ -1,22 +1,22 @@ -import { fetchAsync, logUtils } from '@0x/utils'; +import { fetchAsync } from '@0x/utils'; const EDPS_BASE_URL = 'https://ethereum-dex-prices-service.production.airswap.io'; -const PRICE_BASE_URL = 'https://min-api.cryptocompare.com/data/price?tsyms=USD' +const PRICE_BASE_URL = 'https://min-api.cryptocompare.com/data/price?tsyms=USD'; -export type EdpsResponse = EdpsWrapper[] +export type EdpsResponse = EdpsWrapper[]; export interface EdpsWrapper { - [key: string]: EdpsExchange + [key: string]: EdpsExchange; } export interface EdpsExchange { - exchangeName: string, - totalPrice: number, - tokenAmount: number, - tokenSymbol: string, - avgPrice: number, - timestamp: number, - error: string + exchangeName: string; + totalPrice: number; + tokenAmount: number; + tokenSymbol: string; + avgPrice: number; + timestamp: number; + error: string; } export interface PriceResponse { @@ -34,23 +34,22 @@ export class EdpsSource { const resp = await fetchAsync(edpsUrl); const respJson: EdpsResponse = await resp.json(); const allExchanges = new Map(); - for (let entry of respJson) { - for (let key in entry) { + for (const entry of respJson) { + for (const key of Object.keys(entry)) { allExchanges.set(key, entry[key]); } } return allExchanges; } -} -export class PriceSource { /** - * Call CryptoCompare Price API to get USD price of token. + * + * Call price API to fetch USD price for symbol. */ public async getUsdPriceAsync(symbol: string): Promise { - const priceUrl = `${PRICE_BASE_URL}&fsym=${symbol}` + const priceUrl = `${PRICE_BASE_URL}&fsym=${symbol}`; const resp = await fetchAsync(priceUrl); const respJson: PriceResponse = await resp.json(); return respJson.USD; } -} \ No newline at end of file +} diff --git a/packages/pipeline/src/entities/slippage.ts b/packages/pipeline/src/entities/slippage.ts index d046f382b5..bf330151a2 100644 --- a/packages/pipeline/src/entities/slippage.ts +++ b/packages/pipeline/src/entities/slippage.ts @@ -1,5 +1,7 @@ -import { BigNumber } from '@0x/utils'; import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { BigNumber } from '@0x/utils'; + import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; @Entity({ name: 'slippage', schema: 'raw' }) diff --git a/packages/pipeline/src/parsers/slippage/index.ts b/packages/pipeline/src/parsers/slippage/index.ts index e38fb4c6a7..fb3e3cd2fc 100644 --- a/packages/pipeline/src/parsers/slippage/index.ts +++ b/packages/pipeline/src/parsers/slippage/index.ts @@ -1,34 +1,34 @@ import { BigNumber } from '@0x/utils'; -import * as R from 'ramda'; import { EdpsExchange } from '../../data_sources/slippage'; import { Slippage } from '../../entities'; -import { symbol } from 'prop-types'; /** * Calculates slippage and returns Slippage entity. - * - * @param usdAmount - * @param exchange - * @param buyEdps - * @param sellEdps + * + * @param usdAmount Amount to buy/sell in USD. + * @param exchange Exchange where we are calculating slippage. + * @param buyEdps Ethereum DEX price service object for buy side. + * @param sellEdps Ethereum DEX price service object for sell side. + * */ - export function calculateSlippage(usdAmount: number, exchange: string, - buyEdps: Map, sellEdps: Map) { - const b = buyEdps.get(exchange); - const s = sellEdps.get(exchange); - const slippage = new Slippage(); - if (b && s && b.avgPrice && s.avgPrice) { - slippage.observedTimestamp = b.timestamp; - slippage.symbol = b.tokenSymbol; - slippage.exchange = exchange; - slippage.usdAmount = new BigNumber(usdAmount); - slippage.tokenAmount = new BigNumber(Number(b.tokenAmount)); // API returns a string - slippage.avgPriceInEthBuy = new BigNumber(b.avgPrice); - slippage.avgPriceInEthSell = new BigNumber(s.avgPrice); - slippage.slippage = new BigNumber((b.avgPrice - s.avgPrice) / b.avgPrice); - - } - return slippage; +export function calculateSlippage(usdAmount: number, + exchange: string, + buyEdps: Map, + sellEdps: Map): Slippage { + const b = buyEdps.get(exchange); + const s = sellEdps.get(exchange); + const slippage = new Slippage(); + if (b && s && b.avgPrice && s.avgPrice) { + slippage.observedTimestamp = b.timestamp; + slippage.symbol = b.tokenSymbol; + slippage.exchange = exchange; + slippage.usdAmount = new BigNumber(usdAmount); + slippage.tokenAmount = new BigNumber(Number(b.tokenAmount)); // API returns a string + slippage.avgPriceInEthBuy = new BigNumber(b.avgPrice); + slippage.avgPriceInEthSell = new BigNumber(s.avgPrice); + slippage.slippage = new BigNumber((b.avgPrice - s.avgPrice) / b.avgPrice); } + return slippage; +} diff --git a/packages/pipeline/src/scripts/pull_slippage.ts b/packages/pipeline/src/scripts/pull_slippage.ts index 9411e51389..cb82a5ac96 100644 --- a/packages/pipeline/src/scripts/pull_slippage.ts +++ b/packages/pipeline/src/scripts/pull_slippage.ts @@ -1,19 +1,20 @@ -import * as R from 'ramda'; -import { Connection, ConnectionOptions, createConnection, Repository, PromiseUtils, AdvancedConsoleLogger } from 'typeorm'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + import { logUtils } from '@0x/utils'; -import { EdpsExchange, EdpsSource, PriceResponse, PriceSource } from '../data_sources/slippage'; -import { handleError } from '../utils'; -import { string, number } from 'prop-types'; -import { calculateSlippage } from '../parsers/slippage'; + +import { EdpsSource} from '../data_sources/slippage'; import { Slippage } from '../entities'; import * as ormConfig from '../ormconfig'; +import { calculateSlippage } from '../parsers/slippage'; +import { handleError } from '../utils'; // Number of orders to save at once. const BATCH_SAVE_SIZE = 1000; // USD amounts for slippage depths +// tslint:disable-next-line:custom-no-magic-numbers const USD_AMOUNTS = [10, 100, 1000, 10000]; - + // TODO: fetch from database const TOKENS = ['BAT', 'DAI', 'FUN', 'MANA', 'OMG', 'REP', 'TUSD', 'ZRX', 'MKR', 'BNB', 'USDC']; @@ -21,30 +22,28 @@ let connection: Connection; (async () => { connection = await createConnection(ormConfig as ConnectionOptions); - const priceSource = new PriceSource(); const edpsSource = new EdpsSource(); logUtils.log('Fetching slippage records'); - let nestedSlippages: Slippage[][][] = await Promise.all(await TOKENS.map(async (symbol) => { - const usdPrice = await priceSource.getUsdPriceAsync(symbol); - return Promise.all(USD_AMOUNTS.map(async (usdAmount) => { + const nestedSlippages: Slippage[][][] = await Promise.all(TOKENS.map(async symbol => { + const usdPrice = await edpsSource.getUsdPriceAsync(symbol); + return Promise.all(USD_AMOUNTS.map(async usdAmount => { const amount = usdAmount / usdPrice; const buyEdps = await edpsSource.getEdpsAsync('buy', symbol, amount); const sellEdps = await edpsSource.getEdpsAsync('sell', symbol, amount); - const slippages = Array.from(buyEdps.keys()).map((exchange) => { + return Array.from(buyEdps.keys()).map(exchange => { const slippage: Slippage = calculateSlippage(usdAmount, exchange, buyEdps, sellEdps); return slippage; }); - return slippages; })); })); - let slippagesWithEmptyRecords = await nestedSlippages + const slippagesWithEmptyRecords = nestedSlippages .reduce((acc, val) => acc.concat(val)) .reduce((acc, val) => acc.concat(val)); - let slippages = slippagesWithEmptyRecords.filter((slippage) => slippage.observedTimestamp) + const slippages = slippagesWithEmptyRecords.filter(slippage => slippage.observedTimestamp); const SlippageRepository = connection.getRepository(Slippage); logUtils.log(`Saving ${slippages.length} records to database`); await SlippageRepository.save(slippages, { chunk: Math.ceil(slippages.length / BATCH_SAVE_SIZE) }); - logUtils.log("Done"); + logUtils.log('Done'); process.exit(0); -})().catch(handleError); \ No newline at end of file +})().catch(handleError); diff --git a/packages/pipeline/test/entities/slippage_test.ts b/packages/pipeline/test/entities/slippage_test.ts index 5f95e28c5e..0dc655adeb 100644 --- a/packages/pipeline/test/entities/slippage_test.ts +++ b/packages/pipeline/test/entities/slippage_test.ts @@ -1,6 +1,5 @@ import { BigNumber } from '@0x/utils'; import 'mocha'; -import * as R from 'ramda'; import 'reflect-metadata'; import { Slippage } from '../../src/entities'; @@ -19,7 +18,7 @@ const slippage = { tokenAmount: new BigNumber(25), avgPriceInEthBuy: new BigNumber(0.0022), avgPriceInEthSell: new BigNumber(0.002), - slippage: new BigNumber(0.01) + slippage: new BigNumber(0.01), }; // tslint:disable:custom-no-magic-numbers @@ -28,8 +27,8 @@ describe('Slippage entity', () => { const connection = await createDbConnectionOnceAsync(); const slippages = [slippage]; const slippageRepository = connection.getRepository(Slippage); - for (const slippage of slippages) { - await testSaveAndFindEntityAsync(slippageRepository, slippage); + for (const slippageRecord of slippages) { + await testSaveAndFindEntityAsync(slippageRepository, slippageRecord); } }); }); diff --git a/packages/pipeline/test/parsers/slippage/index_test.ts b/packages/pipeline/test/parsers/slippage/index_test.ts new file mode 100644 index 0000000000..44e2adcf73 --- /dev/null +++ b/packages/pipeline/test/parsers/slippage/index_test.ts @@ -0,0 +1,64 @@ +import { BigNumber } from '@0x/utils'; +import * as chai from 'chai'; +import 'mocha'; + +import { EdpsExchange } from '../../../src/data_sources/slippage'; +import { Slippage } from '../../../src/entities'; +import { calculateSlippage } from '../../../src/parsers/slippage'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('slippage', () => { + describe('calculateSlippage', () => { + it('calculates slippage correctly', () => { + + const exchange = 'Radar Relay'; + const ts = 1549961441473; + const symbol = 'DAI'; + const amount = 1000; + const buyPrice = 10; + const sellPrice = 9; + const expectedSlippage = 0.1; + + const buyEdps = new Map(); + const buyOrder: EdpsExchange = { + exchangeName: exchange, + totalPrice: buyPrice, + tokenAmount: amount, + tokenSymbol: symbol, + avgPrice: buyPrice / amount, + timestamp: ts, + error: '', + }; + buyEdps.set(exchange, buyOrder); + + const sellEdps = new Map(); + const sellOrder: EdpsExchange = { + exchangeName: exchange, + totalPrice: sellPrice, + tokenAmount: amount, + tokenSymbol: symbol, + avgPrice: sellPrice / amount, + timestamp: ts, + error: '', + }; + sellEdps.set(exchange, sellOrder); + const expected = new Slippage(); + expected.observedTimestamp = ts; + expected.symbol = symbol; + expected.exchange = exchange; + expected.usdAmount = new BigNumber(amount); + expected.tokenAmount = new BigNumber(amount); // API returns a string + expected.avgPriceInEthBuy = new BigNumber(buyPrice / amount); + expected.avgPriceInEthSell = new BigNumber(sellPrice / amount); + expected.slippage = new BigNumber(0.1); + + const actual = calculateSlippage(amount, exchange, buyEdps, sellEdps); + const actualSlippage: BigNumber = actual.slippage ? actual.slippage : new BigNumber(0); + expect(actualSlippage.toNumber()).to.be.closeTo(expectedSlippage, 0.0001); + }); + }); +}); From d5f5e7966180022c5fa5b1ce14ea8c7d0ab78346 Mon Sep 17 00:00:00 2001 From: fragosti Date: Tue, 12 Feb 2019 10:41:47 -0800 Subject: [PATCH 05/37] Add radar data source --- .../pipeline/src/data_sources/radar/index.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 packages/pipeline/src/data_sources/radar/index.ts diff --git a/packages/pipeline/src/data_sources/radar/index.ts b/packages/pipeline/src/data_sources/radar/index.ts new file mode 100644 index 0000000000..7f7abb3334 --- /dev/null +++ b/packages/pipeline/src/data_sources/radar/index.ts @@ -0,0 +1,34 @@ +import { fetchAsync, logUtils } from '@0x/utils'; +import { RadarBook, RadarMarket } from '@radarrelay/types'; + +const RADAR_BASE_URL = 'https://api.radarrelay.com/v2/'; +const ACTIVE_MARKETS_URL = `${RADAR_BASE_URL}/markets`; +const MAX_PER_PAGE = 1000; + +export const RADAR_SOURCE = 'radar'; + +// tslint:disable:prefer-function-over-method +// ^ Keep consistency with other sources and help logical organization +export class RadarSource { + /** + * Call Radar API to find out which markets they are maintaining orderbooks for. + */ + public async getActiveMarketsAsync(): Promise { + logUtils.log('Getting all active Radar markets'); + const resp = await fetchAsync(`${ACTIVE_MARKETS_URL}?perPage=${MAX_PER_PAGE}`); + const markets: RadarMarket[] = await resp.json(); + logUtils.log(`Got ${markets.length} markets.`); + return markets; + } + + /** + * Retrieve orderbook from Radar API for a given market. + * @param marketId String identifying the market we want data for. Eg. 'REP/AUG' + */ + public async getMarketOrderbookAsync(marketId: string): Promise { + logUtils.log(`${marketId}: Retrieving orderbook.`); + const marketOrderbookUrl = `${ACTIVE_MARKETS_URL}/${marketId}/book`; + const resp = await fetchAsync(marketOrderbookUrl); + return resp.json(); + } +} From f3a537d5c24759f2ae772f7bd838adb47967556e Mon Sep 17 00:00:00 2001 From: fragosti Date: Tue, 12 Feb 2019 14:35:50 -0800 Subject: [PATCH 06/37] Add radar parser --- .../src/parsers/radar_orders/index.ts | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 packages/pipeline/src/parsers/radar_orders/index.ts diff --git a/packages/pipeline/src/parsers/radar_orders/index.ts b/packages/pipeline/src/parsers/radar_orders/index.ts new file mode 100644 index 0000000000..b4f7e47688 --- /dev/null +++ b/packages/pipeline/src/parsers/radar_orders/index.ts @@ -0,0 +1,89 @@ +import { BigNumber } from '@0x/utils'; +import { Web3Wrapper } from '@0x/web3-wrapper'; +import { RadarBook, RadarMarket, RadarOrderType, RadarSignedOrder } from '@radarrelay/types'; +import * as R from 'ramda'; + +import { aggregateOrders, GenericRawOrder } from '../utils'; + +import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; +import { OrderType } from '../../types'; + +/** + * Marque function of this file. + * 1) Takes in orders from an orderbook, + * other information attached. + * @param radarOrderbook A raw orderbook that we pull from the radar API. + * @param radarMarket An object containing market data also directly from the API. + * @param observedTimestamp Time at which the orders for the market were pulled. + * @param source The exchange where these orders are placed. In this case 'radar'. + */ +export function parseRadarOrders( + radarOrderbook: RadarBook, + radarMarket: RadarMarket, + observedTimestamp: number, + source: string, +): TokenOrder[] { + const transformToGeneric = (radarOrder: RadarSignedOrder) => _toGeneric(radarMarket, radarOrder); + const aggregatedBids = aggregateOrders(_removeUndefined(R.map(transformToGeneric, radarOrderbook.bids))); + const aggregatedAsks = aggregateOrders(_removeUndefined(R.map(transformToGeneric, radarOrderbook.asks))); + const parsedBids = aggregatedBids.map(order => + parseRadarOrder(radarMarket, observedTimestamp, OrderType.Bid, source, order), + ); + const parsedAsks = aggregatedAsks.map(order => + parseRadarOrder(radarMarket, observedTimestamp, OrderType.Ask, source, order), + ); + return parsedBids.concat(parsedAsks); +} + +/** + * Parse a single aggregated radar order in order to form a tokenOrder entity + * which can be saved into the database. + * @param radarMarket An object containing information about the market where these + * trades have been placed. + * @param observedTimestamp The time when the API response returned back to us. + * @param orderType 'bid' or 'ask' enum. + * @param source Exchange where these orders were placed. + * @param radarOrder A tuple which we will convert to volume-basis. + */ +export function parseRadarOrder( + radarMarket: RadarMarket, + observedTimestamp: number, + orderType: OrderType, + source: string, + radarOrder: [string, BigNumber], +): TokenOrder { + const tokenOrder = new TokenOrder(); + const price = new BigNumber(radarOrder[0]); + const amount = radarOrder[1]; + const splitId = radarMarket.id.split('-'); + + tokenOrder.source = source; + tokenOrder.observedTimestamp = observedTimestamp; + tokenOrder.orderType = orderType; + tokenOrder.price = price; + + tokenOrder.baseAssetSymbol = splitId[0]; + tokenOrder.baseAssetAddress = radarMarket.baseTokenAddress || null; + tokenOrder.baseVolume = amount; + + tokenOrder.quoteAssetSymbol = splitId[1]; + tokenOrder.quoteAssetAddress = radarMarket.quoteTokenAddress || null; + tokenOrder.quoteVolume = price.times(amount); + return tokenOrder; +} + +function _toGeneric(radarMarket: RadarMarket, radarOrder: RadarSignedOrder): GenericRawOrder | undefined { + if (radarMarket.baseTokenDecimals === undefined) { + return undefined; + } + const rawAmount = + radarOrder.type === RadarOrderType.ASK + ? radarOrder.signedOrder.makerAssetAmount + : radarOrder.signedOrder.takerAssetAmount; + return { + price: radarOrder.price.toString(), + amount: Web3Wrapper.toUnitAmount(new BigNumber(rawAmount.toString()), radarMarket.baseTokenDecimals).toString(), + }; +} + +const _removeUndefined = R.reject(R.isNil); From 4db9b8b0e34e6eec7ec445ebd93bbf8fff3a3bbd Mon Sep 17 00:00:00 2001 From: fragosti Date: Tue, 12 Feb 2019 14:49:32 -0800 Subject: [PATCH 07/37] Add script for pulling radar orderbook --- packages/pipeline/package.json | 1 + .../pipeline/src/data_sources/radar/index.ts | 2 +- .../scripts/pull_radar_orderbook_snapshots.ts | 56 ++++++++++++++++++ yarn.lock | 57 +++++++++++++++---- 4 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index b2ad39a5f2..ecb53e5da3 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -49,6 +49,7 @@ "@0x/types": "^2.0.2", "@0x/utils": "^4.0.3", "@0x/web3-wrapper": "^4.0.2", + "@radarrelay/types": "^1.2.1", "@types/dockerode": "^2.5.9", "@types/p-limit": "^2.0.0", "async-parallel": "^1.2.3", diff --git a/packages/pipeline/src/data_sources/radar/index.ts b/packages/pipeline/src/data_sources/radar/index.ts index 7f7abb3334..bbc55eef99 100644 --- a/packages/pipeline/src/data_sources/radar/index.ts +++ b/packages/pipeline/src/data_sources/radar/index.ts @@ -27,7 +27,7 @@ export class RadarSource { */ public async getMarketOrderbookAsync(marketId: string): Promise { logUtils.log(`${marketId}: Retrieving orderbook.`); - const marketOrderbookUrl = `${ACTIVE_MARKETS_URL}/${marketId}/book`; + const marketOrderbookUrl = `${ACTIVE_MARKETS_URL}/${marketId}/book?perPage=${MAX_PER_PAGE}`; const resp = await fetchAsync(marketOrderbookUrl); return resp.json(); } diff --git a/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts new file mode 100644 index 0000000000..91d33d2cc9 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts @@ -0,0 +1,56 @@ +import { logUtils } from '@0x/utils'; +import { RadarMarket } from '@radarrelay/types'; +import * as R from 'ramda'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { RADAR_SOURCE, RadarSource } from '../data_sources/radar'; +import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseRadarOrders } from '../parsers/radar_orders'; +import { handleError } from '../utils'; + +// Number of orders to save at once. +const BATCH_SAVE_SIZE = 1000; + +// Number of markets to retrieve orderbooks for at once. +const MARKET_ORDERBOOK_REQUEST_BATCH_SIZE = 50; + +// Delay between market orderbook requests. +const MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY = 5000; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + const radarSource = new RadarSource(); + const markets = await radarSource.getActiveMarketsAsync(); + for (const marketsChunk of R.splitEvery(MARKET_ORDERBOOK_REQUEST_BATCH_SIZE, markets)) { + await Promise.all( + marketsChunk.map(async (market: RadarMarket) => getAndSaveMarketOrderbookAsync(radarSource, market)), + ); + await new Promise(resolve => setTimeout(resolve, MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY)); + } + process.exit(0); +})().catch(handleError); + +/** + * Retrieve orderbook from radar API for a given market. Parse orders and insert + * them into our database. + * @param radarSource Data source which can query radar API. + * @param market Object from radar API containing market data. + */ +async function getAndSaveMarketOrderbookAsync(radarSource: RadarSource, market: RadarMarket): Promise { + const orderBook = await radarSource.getMarketOrderbookAsync(market.id); + const observedTimestamp = Date.now(); + + logUtils.log(`${market.id}: Parsing orders.`); + const orders = parseRadarOrders(orderBook, market, observedTimestamp, RADAR_SOURCE); + + if (orders.length > 0) { + logUtils.log(`${market.id}: Saving ${orders.length} orders.`); + const TokenOrderRepository = connection.getRepository(TokenOrder); + await TokenOrderRepository.save(orders, { chunk: Math.ceil(orders.length / BATCH_SAVE_SIZE) }); + } else { + logUtils.log(`${market.id}: 0 orders to save.`); + } +} diff --git a/yarn.lock b/yarn.lock index fae178e3e4..444b5eeed7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -593,6 +593,14 @@ dependencies: npm-registry-client "7.0.9" +"@0xproject/types@^1.0.1-rc.3": + version "1.1.4" + resolved "https://registry.npmjs.org/@0xproject/types/-/types-1.1.4.tgz#3ffd65e670d6a21dab19ee0ffd5fad0056291b8e" + dependencies: + "@types/node" "*" + bignumber.js "~4.1.0" + ethereum-types "^1.0.11" + "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.0.0-beta.35": version "7.0.0" resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz#06e2ab19bdb535385559aabb5ba59729482800f8" @@ -1267,6 +1275,13 @@ call-me-maybe "^1.0.1" glob-to-regexp "^0.3.0" +"@radarrelay/types@^1.2.1": + version "1.2.1" + resolved "https://registry.npmjs.org/@radarrelay/types/-/types-1.2.1.tgz#d16edb43d0735a31c887b9e79ff6e53924ac8cc5" + dependencies: + "@0xproject/types" "^1.0.1-rc.3" + bignumber.js "^5.0.0" + "@reach/component-component@^0.1.1": version "0.1.1" resolved "https://registry.yarnpkg.com/@reach/component-component/-/component-component-0.1.1.tgz#62ea2ec290da32f5e3a9872fb51f9a3ae4370cc4" @@ -3432,6 +3447,10 @@ bignumber.js@7.2.1: version "7.2.1" resolved "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz#80c048759d826800807c4bfd521e50edbba57a5f" +bignumber.js@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/bignumber.js/-/bignumber.js-5.0.0.tgz#fbce63f09776b3000a83185badcde525daf34833" + "bignumber.js@git+https://github.com/debris/bignumber.js.git#94d7146671b9719e00a09c29b01a691bc85048c2": version "2.0.7" resolved "git+https://github.com/debris/bignumber.js.git#94d7146671b9719e00a09c29b01a691bc85048c2" @@ -6404,6 +6423,13 @@ ethereum-common@^0.0.18: version "0.0.18" resolved "https://registry.yarnpkg.com/ethereum-common/-/ethereum-common-0.0.18.tgz#2fdc3576f232903358976eb39da783213ff9523f" +ethereum-types@^1.0.11: + version "1.1.6" + resolved "https://registry.npmjs.org/ethereum-types/-/ethereum-types-1.1.6.tgz#14437dbf401de361e70dac6358e5f2915ad3c35d" + dependencies: + "@types/node" "*" + bignumber.js "~4.1.0" + ethereumjs-abi@0.6.5: version "0.6.5" resolved "https://registry.yarnpkg.com/ethereumjs-abi/-/ethereumjs-abi-0.6.5.tgz#5a637ef16ab43473fa72a29ad90871405b3f5241" @@ -7910,9 +7936,19 @@ got@^6.7.1: unzip-response "^2.0.1" url-parse-lax "^1.0.0" -graceful-fs@4.1.15, graceful-fs@^3.0.0, graceful-fs@^4.0.0, graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9, graceful-fs@~1.2.0: +graceful-fs@^3.0.0: + version "3.0.11" + resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-3.0.11.tgz#7613c778a1afea62f25c630a086d7f3acbbdd818" + dependencies: + natives "^1.1.0" + +graceful-fs@^4.0.0, graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9: version "4.1.15" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00" + resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00" + +graceful-fs@~1.2.0: + version "1.2.3" + resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-1.2.3.tgz#15a4806a57547cb2d2dbf27f42e89a8c3451b364" "graceful-readlink@>= 1.0.0": version "1.0.1" @@ -11343,6 +11379,10 @@ nanomatch@^1.2.9: snapdragon "^0.8.1" to-regex "^3.0.1" +natives@^1.1.0: + version "1.1.6" + resolved "https://registry.npmjs.org/natives/-/natives-1.1.6.tgz#a603b4a498ab77173612b9ea1acdec4d980f00bb" + natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" @@ -13463,8 +13503,7 @@ react-dom@^16.3.2: react-dom@^16.4.2: version "16.8.1" - resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-16.8.1.tgz#ec860f98853d09d39bafd3a6f1e12389d283dbb4" - integrity sha512-N74IZUrPt6UiDjXaO7UbDDFXeUXnVhZzeRLy/6iqqN1ipfjrhR60Bp5NuBK+rv3GMdqdIuwIl22u1SYwf330bg== + resolved "https://registry.npmjs.org/react-dom/-/react-dom-16.8.1.tgz#ec860f98853d09d39bafd3a6f1e12389d283dbb4" dependencies: loose-envify "^1.1.0" object-assign "^4.1.1" @@ -13528,8 +13567,8 @@ react-highlight@0xproject/react-highlight#react-peer-deps: dependencies: highlight.js "^9.11.0" highlightjs-solidity "^1.0.5" - react "^16.5.2" - react-dom "^16.5.2" + react "^16.4.2" + react-dom "^16.4.2" react-hot-loader@^4.3.3: version "4.3.4" @@ -13776,8 +13815,7 @@ react@^16.3.2: react@^16.4.2: version "16.8.1" - resolved "https://registry.yarnpkg.com/react/-/react-16.8.1.tgz#ae11831f6cb2a05d58603a976afc8a558e852c4a" - integrity sha512-wLw5CFGPdo7p/AgteFz7GblI2JPOos0+biSoxf1FPsGxWQZdN/pj6oToJs1crn61DL3Ln7mN86uZ4j74p31ELQ== + resolved "https://registry.npmjs.org/react/-/react-16.8.1.tgz#ae11831f6cb2a05d58603a976afc8a558e852c4a" dependencies: loose-envify "^1.1.0" object-assign "^4.1.1" @@ -14657,8 +14695,7 @@ schedule@^0.5.0: scheduler@^0.13.1: version "0.13.1" - resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.13.1.tgz#1a217df1bfaabaf4f1b92a9127d5d732d85a9591" - integrity sha512-VJKOkiKIN2/6NOoexuypwSrybx13MY7NSy9RNt8wPvZDMRT1CW6qlpF5jXRToXNHz3uWzbm2elNpZfXfGPqP9A== + resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.13.1.tgz#1a217df1bfaabaf4f1b92a9127d5d732d85a9591" dependencies: loose-envify "^1.1.0" object-assign "^4.1.1" From d79c7f70be33334d45a3ad6664d773188c60646d Mon Sep 17 00:00:00 2001 From: fragosti Date: Tue, 12 Feb 2019 14:58:05 -0800 Subject: [PATCH 08/37] fix linting error --- packages/pipeline/src/parsers/radar_orders/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/pipeline/src/parsers/radar_orders/index.ts b/packages/pipeline/src/parsers/radar_orders/index.ts index b4f7e47688..eec67fe5e4 100644 --- a/packages/pipeline/src/parsers/radar_orders/index.ts +++ b/packages/pipeline/src/parsers/radar_orders/index.ts @@ -83,7 +83,9 @@ function _toGeneric(radarMarket: RadarMarket, radarOrder: RadarSignedOrder): Gen return { price: radarOrder.price.toString(), amount: Web3Wrapper.toUnitAmount(new BigNumber(rawAmount.toString()), radarMarket.baseTokenDecimals).toString(), + // TODO: Add remainingFillableAmount since its available }; } +// tslint:disable-next-line:no-unbound-method const _removeUndefined = R.reject(R.isNil); From c36e1ad056053bbfad4dd6fe93a72f75f4133e2b Mon Sep 17 00:00:00 2001 From: askeluv Date: Wed, 13 Feb 2019 20:23:07 +0800 Subject: [PATCH 09/37] Changes made as per comments - still have two minor things to fix --- .../1549856835629-CreateSlippageTable.ts | 4 +- .../{slippage => dex_prices}/index.ts | 16 ------- .../ohlcv_external/crypto_compare.ts | 11 +++++ packages/pipeline/src/entities/slippage.ts | 8 ++-- .../pipeline/src/parsers/slippage/index.ts | 12 +++--- .../pipeline/src/scripts/pull_slippage.ts | 42 ++++++++++++------- .../test/parsers/slippage/index_test.ts | 3 +- 7 files changed, 50 insertions(+), 46 deletions(-) rename packages/pipeline/src/data_sources/{slippage => dex_prices}/index.ts (71%) diff --git a/packages/pipeline/migrations/1549856835629-CreateSlippageTable.ts b/packages/pipeline/migrations/1549856835629-CreateSlippageTable.ts index d9d6ff4f61..7dfe9fc7fe 100644 --- a/packages/pipeline/migrations/1549856835629-CreateSlippageTable.ts +++ b/packages/pipeline/migrations/1549856835629-CreateSlippageTable.ts @@ -1,4 +1,4 @@ -import {MigrationInterface, QueryRunner, Table} from "typeorm"; +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; const slippage = new Table({ name: 'raw.slippage', @@ -16,7 +16,6 @@ const slippage = new Table({ }); export class CreateSlippageTable1549856835629 implements MigrationInterface { - public async up(queryRunner: QueryRunner): Promise { await queryRunner.createTable(slippage); } @@ -24,5 +23,4 @@ export class CreateSlippageTable1549856835629 implements MigrationInterface { public async down(queryRunner: QueryRunner): Promise { await queryRunner.dropTable(slippage); } - } diff --git a/packages/pipeline/src/data_sources/slippage/index.ts b/packages/pipeline/src/data_sources/dex_prices/index.ts similarity index 71% rename from packages/pipeline/src/data_sources/slippage/index.ts rename to packages/pipeline/src/data_sources/dex_prices/index.ts index 65bb8fe046..afa4db6953 100644 --- a/packages/pipeline/src/data_sources/slippage/index.ts +++ b/packages/pipeline/src/data_sources/dex_prices/index.ts @@ -1,7 +1,6 @@ import { fetchAsync } from '@0x/utils'; const EDPS_BASE_URL = 'https://ethereum-dex-prices-service.production.airswap.io'; -const PRICE_BASE_URL = 'https://min-api.cryptocompare.com/data/price?tsyms=USD'; export type EdpsResponse = EdpsWrapper[]; @@ -19,10 +18,6 @@ export interface EdpsExchange { error: string; } -export interface PriceResponse { - USD: number; -} - // tslint:disable:prefer-function-over-method // ^ Keep consistency with other sources and help logical organization export class EdpsSource { @@ -41,15 +36,4 @@ export class EdpsSource { } return allExchanges; } - - /** - * - * Call price API to fetch USD price for symbol. - */ - public async getUsdPriceAsync(symbol: string): Promise { - const priceUrl = `${PRICE_BASE_URL}&fsym=${symbol}`; - const resp = await fetchAsync(priceUrl); - const respJson: PriceResponse = await resp.json(); - return respJson.USD; - } } diff --git a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts index 85042501b1..526fc5f6bb 100644 --- a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts +++ b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts @@ -33,6 +33,10 @@ export interface CryptoCompareOHLCVParams { toTs?: number; } +export interface CryptoCompareUsdPrice { + USD: number; +} + const ONE_HOUR = 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers const ONE_SECOND = 1000; const ONE_HOUR_AGO = new Date().getTime() - ONE_HOUR; @@ -96,6 +100,13 @@ export class CryptoCompareOHLCVSource { }; return R.unfold(f, pair); } + + public async getUsdPriceAsync(symbol: string): Promise { + const priceUrl = `https://min-api.cryptocompare.com/data/price?tsyms=USD&fsym=${symbol}`; + const resp = await fetchAsync(priceUrl); + const respJson: CryptoCompareUsdPrice = await resp.json(); + return respJson.USD; + } } function hasData(record: CryptoCompareOHLCVRecord): boolean { diff --git a/packages/pipeline/src/entities/slippage.ts b/packages/pipeline/src/entities/slippage.ts index bf330151a2..87744ca048 100644 --- a/packages/pipeline/src/entities/slippage.ts +++ b/packages/pipeline/src/entities/slippage.ts @@ -6,7 +6,7 @@ import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; @Entity({ name: 'slippage', schema: 'raw' }) export class Slippage { - @PrimaryColumn({ name: 'observed_timestamp', type: 'bigint', transformer: numberToBigIntTransformer}) + @PrimaryColumn({ name: 'observed_timestamp', type: 'bigint', transformer: numberToBigIntTransformer }) public observedTimestamp!: number; @PrimaryColumn({ name: 'symbol' }) public symbol!: string; @@ -14,11 +14,11 @@ export class Slippage { public exchange!: string; @PrimaryColumn({ name: 'usd_amount', type: 'numeric', transformer: bigNumberTransformer }) public usdAmount!: BigNumber; - @PrimaryColumn({ name: 'token_amount', type: 'numeric', transformer: bigNumberTransformer }) + @Column({ name: 'token_amount', type: 'numeric', transformer: bigNumberTransformer }) public tokenAmount!: BigNumber; - @PrimaryColumn({ name: 'avg_price_in_eth_sell', type: 'numeric', transformer: bigNumberTransformer }) + @Column({ name: 'avg_price_in_eth_sell', type: 'numeric', transformer: bigNumberTransformer }) public avgPriceInEthSell?: BigNumber; - @PrimaryColumn({ name: 'avg_price_in_eth_buy', type: 'numeric', transformer: bigNumberTransformer }) + @Column({ name: 'avg_price_in_eth_buy', type: 'numeric', transformer: bigNumberTransformer }) public avgPriceInEthBuy?: BigNumber; @Column({ name: 'slippage', type: 'numeric', transformer: bigNumberTransformer }) public slippage?: BigNumber; diff --git a/packages/pipeline/src/parsers/slippage/index.ts b/packages/pipeline/src/parsers/slippage/index.ts index fb3e3cd2fc..7f5904fb3d 100644 --- a/packages/pipeline/src/parsers/slippage/index.ts +++ b/packages/pipeline/src/parsers/slippage/index.ts @@ -1,6 +1,6 @@ import { BigNumber } from '@0x/utils'; -import { EdpsExchange } from '../../data_sources/slippage'; +import { EdpsExchange } from '../../data_sources/dex_prices'; import { Slippage } from '../../entities'; /** @@ -13,10 +13,12 @@ import { Slippage } from '../../entities'; * */ -export function calculateSlippage(usdAmount: number, - exchange: string, - buyEdps: Map, - sellEdps: Map): Slippage { +export function calculateSlippage( + usdAmount: number, + exchange: string, + buyEdps: Map, + sellEdps: Map, +): Slippage { const b = buyEdps.get(exchange); const s = sellEdps.get(exchange); const slippage = new Slippage(); diff --git a/packages/pipeline/src/scripts/pull_slippage.ts b/packages/pipeline/src/scripts/pull_slippage.ts index cb82a5ac96..0a0bb3360b 100644 --- a/packages/pipeline/src/scripts/pull_slippage.ts +++ b/packages/pipeline/src/scripts/pull_slippage.ts @@ -1,8 +1,10 @@ +import * as R from 'ramda'; import { Connection, ConnectionOptions, createConnection } from 'typeorm'; import { logUtils } from '@0x/utils'; -import { EdpsSource} from '../data_sources/slippage'; +import { CryptoCompareOHLCVSource } from '../data_sources/ohlcv_external/crypto_compare'; +import { EdpsSource } from '../data_sources/dex_prices'; import { Slippage } from '../entities'; import * as ormConfig from '../ormconfig'; import { calculateSlippage } from '../parsers/slippage'; @@ -11,35 +13,43 @@ import { handleError } from '../utils'; // Number of orders to save at once. const BATCH_SAVE_SIZE = 1000; +// Maximum requests per second to CryptoCompare +const CRYPTO_COMPARE_MAX_REQS_PER_SECOND = 60; + // USD amounts for slippage depths // tslint:disable-next-line:custom-no-magic-numbers const USD_AMOUNTS = [10, 100, 1000, 10000]; // TODO: fetch from database -const TOKENS = ['BAT', 'DAI', 'FUN', 'MANA', 'OMG', 'REP', 'TUSD', 'ZRX', 'MKR', 'BNB', 'USDC']; +const TOKENS = ['BAT', 'DAI', 'FUN', 'MANA', 'OMG', 'REP', 'TUSD', 'ZRX', 'MKR', 'BNB', 'USDC', 'LOOM', 'DNT', 'CVC']; let connection: Connection; (async () => { connection = await createConnection(ormConfig as ConnectionOptions); const edpsSource = new EdpsSource(); + const cryptoCompareSource = new CryptoCompareOHLCVSource(CRYPTO_COMPARE_MAX_REQS_PER_SECOND); logUtils.log('Fetching slippage records'); - const nestedSlippages: Slippage[][][] = await Promise.all(TOKENS.map(async symbol => { - const usdPrice = await edpsSource.getUsdPriceAsync(symbol); - return Promise.all(USD_AMOUNTS.map(async usdAmount => { - const amount = usdAmount / usdPrice; - const buyEdps = await edpsSource.getEdpsAsync('buy', symbol, amount); - const sellEdps = await edpsSource.getEdpsAsync('sell', symbol, amount); - return Array.from(buyEdps.keys()).map(exchange => { - const slippage: Slippage = calculateSlippage(usdAmount, exchange, buyEdps, sellEdps); - return slippage; - }); - })); - })); + const nestedSlippages: Slippage[][][] = await Promise.all( + TOKENS.map(async symbol => { + const usdPrice = await cryptoCompareSource.getUsdPriceAsync(symbol); + return Promise.all( + USD_AMOUNTS.map(async usdAmount => { + const amount = usdAmount / usdPrice; + const buyEdps = await edpsSource.getEdpsAsync('buy', symbol, amount); + const sellEdps = await edpsSource.getEdpsAsync('sell', symbol, amount); + return Array.from(buyEdps.keys()).map(exchange => { + const slippage: Slippage = calculateSlippage(usdAmount, exchange, buyEdps, sellEdps); + return slippage; + }); + }), + ); + }), + ); const slippagesWithEmptyRecords = nestedSlippages - .reduce((acc, val) => acc.concat(val)) - .reduce((acc, val) => acc.concat(val)); + .reduce((acc, val) => acc.concat(val)) + .reduce((acc, val) => acc.concat(val)); const slippages = slippagesWithEmptyRecords.filter(slippage => slippage.observedTimestamp); const SlippageRepository = connection.getRepository(Slippage); logUtils.log(`Saving ${slippages.length} records to database`); diff --git a/packages/pipeline/test/parsers/slippage/index_test.ts b/packages/pipeline/test/parsers/slippage/index_test.ts index 44e2adcf73..ce205b1698 100644 --- a/packages/pipeline/test/parsers/slippage/index_test.ts +++ b/packages/pipeline/test/parsers/slippage/index_test.ts @@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils'; import * as chai from 'chai'; import 'mocha'; -import { EdpsExchange } from '../../../src/data_sources/slippage'; +import { EdpsExchange } from '../../../src/data_sources/dex_prices'; import { Slippage } from '../../../src/entities'; import { calculateSlippage } from '../../../src/parsers/slippage'; import { chaiSetup } from '../../utils/chai_setup'; @@ -14,7 +14,6 @@ const expect = chai.expect; describe('slippage', () => { describe('calculateSlippage', () => { it('calculates slippage correctly', () => { - const exchange = 'Radar Relay'; const ts = 1549961441473; const symbol = 'DAI'; From d1cc08f1d90cbeb7a2bb2861e4c2b012c36b6509 Mon Sep 17 00:00:00 2001 From: fragosti Date: Wed, 13 Feb 2019 11:03:40 -0800 Subject: [PATCH 10/37] Attempt to add maker_address token_orderbook_snapshots --- ...-TokenOrderBookSnapshotsAddMakerAddress.ts | 27 ++++++++++ packages/pipeline/src/entities/token_order.ts | 2 + .../src/parsers/radar_orders/index.ts | 50 ++++++++++++++++--- .../test/entities/token_order_test.ts | 1 + 4 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 packages/pipeline/migrations/1550017139695-TokenOrderBookSnapshotsAddMakerAddress.ts diff --git a/packages/pipeline/migrations/1550017139695-TokenOrderBookSnapshotsAddMakerAddress.ts b/packages/pipeline/migrations/1550017139695-TokenOrderBookSnapshotsAddMakerAddress.ts new file mode 100644 index 0000000000..fb13c91f28 --- /dev/null +++ b/packages/pipeline/migrations/1550017139695-TokenOrderBookSnapshotsAddMakerAddress.ts @@ -0,0 +1,27 @@ +import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm'; + +const TOKEN_ORDERBOOK_SNAPSHOT_TABLE = 'raw.token_orderbook_snapshots'; +const NEW_COLUMN_NAME = 'maker_address'; + +export class TokenOrderBookSnapshotsAddMakerAddress1550017139695 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + const snapshotTable = await queryRunner.getTable(TOKEN_ORDERBOOK_SNAPSHOT_TABLE); + if (snapshotTable) { + await queryRunner.addColumn( + TOKEN_ORDERBOOK_SNAPSHOT_TABLE, + new TableColumn({ + name: NEW_COLUMN_NAME, + type: 'varchar', + isPrimary: true, + }), + ); + } + } + + public async down(queryRunner: QueryRunner): Promise { + const snapshotTable = await queryRunner.getTable(TOKEN_ORDERBOOK_SNAPSHOT_TABLE); + if (snapshotTable) { + await queryRunner.dropColumn(snapshotTable, NEW_COLUMN_NAME); + } + } +} diff --git a/packages/pipeline/src/entities/token_order.ts b/packages/pipeline/src/entities/token_order.ts index 2709747cb3..25baca407d 100644 --- a/packages/pipeline/src/entities/token_order.ts +++ b/packages/pipeline/src/entities/token_order.ts @@ -15,6 +15,8 @@ export class TokenOrderbookSnapshot { public price!: BigNumber; @PrimaryColumn({ name: 'base_asset_symbol' }) public baseAssetSymbol!: string; + @PrimaryColumn({ type: String, name: 'maker_address', default: 'unknown' }) + public makerAddress!: string | null; @Column({ nullable: true, type: String, name: 'base_asset_address' }) public baseAssetAddress!: string | null; @Column({ name: 'base_volume', type: 'numeric', transformer: bigNumberTransformer }) diff --git a/packages/pipeline/src/parsers/radar_orders/index.ts b/packages/pipeline/src/parsers/radar_orders/index.ts index eec67fe5e4..47d704a737 100644 --- a/packages/pipeline/src/parsers/radar_orders/index.ts +++ b/packages/pipeline/src/parsers/radar_orders/index.ts @@ -1,3 +1,4 @@ +import { ObjectMap } from '@0x/types'; import { BigNumber } from '@0x/utils'; import { Web3Wrapper } from '@0x/web3-wrapper'; import { RadarBook, RadarMarket, RadarOrderType, RadarSignedOrder } from '@radarrelay/types'; @@ -8,6 +9,12 @@ import { aggregateOrders, GenericRawOrder } from '../utils'; import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; +export interface AggregateOrdersByMaker { + makerAddress: string; + price: string; + amount: BigNumber; +} + /** * Marque function of this file. * 1) Takes in orders from an orderbook, @@ -23,9 +30,8 @@ export function parseRadarOrders( observedTimestamp: number, source: string, ): TokenOrder[] { - const transformToGeneric = (radarOrder: RadarSignedOrder) => _toGeneric(radarMarket, radarOrder); - const aggregatedBids = aggregateOrders(_removeUndefined(R.map(transformToGeneric, radarOrderbook.bids))); - const aggregatedAsks = aggregateOrders(_removeUndefined(R.map(transformToGeneric, radarOrderbook.asks))); + const aggregatedBids = _aggregateOrdersByMaker(radarMarket, radarOrderbook.bids); + const aggregatedAsks = _aggregateOrdersByMaker(radarMarket, radarOrderbook.asks); const parsedBids = aggregatedBids.map(order => parseRadarOrder(radarMarket, observedTimestamp, OrderType.Bid, source, order), ); @@ -43,18 +49,18 @@ export function parseRadarOrders( * @param observedTimestamp The time when the API response returned back to us. * @param orderType 'bid' or 'ask' enum. * @param source Exchange where these orders were placed. - * @param radarOrder A tuple which we will convert to volume-basis. + * @param aggregateOrder An AggregateOrdersByMaker instance which we will convert to volume-basis. */ export function parseRadarOrder( radarMarket: RadarMarket, observedTimestamp: number, orderType: OrderType, source: string, - radarOrder: [string, BigNumber], + aggregateOrder: AggregateOrdersByMaker, ): TokenOrder { const tokenOrder = new TokenOrder(); - const price = new BigNumber(radarOrder[0]); - const amount = radarOrder[1]; + const price = new BigNumber(aggregateOrder.price); + const amount = aggregateOrder.amount; const splitId = radarMarket.id.split('-'); tokenOrder.source = source; @@ -69,6 +75,8 @@ export function parseRadarOrder( tokenOrder.quoteAssetSymbol = splitId[1]; tokenOrder.quoteAssetAddress = radarMarket.quoteTokenAddress || null; tokenOrder.quoteVolume = price.times(amount); + + tokenOrder.makerAddress = aggregateOrder.makerAddress; return tokenOrder; } @@ -83,9 +91,35 @@ function _toGeneric(radarMarket: RadarMarket, radarOrder: RadarSignedOrder): Gen return { price: radarOrder.price.toString(), amount: Web3Wrapper.toUnitAmount(new BigNumber(rawAmount.toString()), radarMarket.baseTokenDecimals).toString(), - // TODO: Add remainingFillableAmount since its available }; } +function _aggregateOrdersByMaker(radarMarket: RadarMarket, radarOrders: RadarSignedOrder[]): AggregateOrdersByMaker[] { + // group all orders by their maker + const ordersByMaker: ObjectMap = radarOrders.reduce( + (acc: ObjectMap, val: RadarSignedOrder) => { + const makerAddress = val.signedOrder.makerAddress; + if (acc[makerAddress]) { + acc[makerAddress].push(val); + } else { + acc[makerAddress] = []; + } + return acc; + }, + {}, + ); + const transformToGeneric = (radarOrder: RadarSignedOrder) => _toGeneric(radarMarket, radarOrder); + const aggregationTuples: AggregateOrdersByMaker[][] = (R.keys(ordersByMaker) as string[]).map((maker: string) => { + const generalizedOrders = _removeUndefined(R.map(transformToGeneric, ordersByMaker[maker])); + const aggregatedOrders = aggregateOrders(generalizedOrders); + return aggregatedOrders.map((order: [string, BigNumber]) => ({ + makerAddress: maker, + price: order[0], + amount: order[1], + })); + }); + return R.unnest(aggregationTuples); +} + // tslint:disable-next-line:no-unbound-method const _removeUndefined = R.reject(R.isNil); diff --git a/packages/pipeline/test/entities/token_order_test.ts b/packages/pipeline/test/entities/token_order_test.ts index c6057f5aae..2101436575 100644 --- a/packages/pipeline/test/entities/token_order_test.ts +++ b/packages/pipeline/test/entities/token_order_test.ts @@ -20,6 +20,7 @@ const tokenOrderbookSnapshot: TokenOrderbookSnapshot = { quoteAssetSymbol: 'ABC', quoteAssetAddress: '0x00923b9a074762b93650716333b3e1473a15048e', quoteVolume: new BigNumber(12.3234234), + makerAddress: null, }; describe('TokenOrderbookSnapshot entity', () => { From 1cb7e70b420934f617810bad2523b25ffd2b50f9 Mon Sep 17 00:00:00 2001 From: fragosti Date: Wed, 13 Feb 2019 13:31:45 -0800 Subject: [PATCH 11/37] increment MAX_PER_PAGE to 10000 --- .../pipeline/src/data_sources/radar/index.ts | 2 +- yarn.lock | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/pipeline/src/data_sources/radar/index.ts b/packages/pipeline/src/data_sources/radar/index.ts index bbc55eef99..d6907a2d24 100644 --- a/packages/pipeline/src/data_sources/radar/index.ts +++ b/packages/pipeline/src/data_sources/radar/index.ts @@ -3,7 +3,7 @@ import { RadarBook, RadarMarket } from '@radarrelay/types'; const RADAR_BASE_URL = 'https://api.radarrelay.com/v2/'; const ACTIVE_MARKETS_URL = `${RADAR_BASE_URL}/markets`; -const MAX_PER_PAGE = 1000; +const MAX_PER_PAGE = 10000; export const RADAR_SOURCE = 'radar'; diff --git a/yarn.lock b/yarn.lock index b33ea8a8cf..b596cab7a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13500,6 +13500,15 @@ react-dom@^16.3.2: object-assign "^4.1.1" prop-types "^15.6.0" +react-dom@^16.4.2: + version "16.8.1" + resolved "https://registry.npmjs.org/react-dom/-/react-dom-16.8.1.tgz#ec860f98853d09d39bafd3a6f1e12389d283dbb4" + dependencies: + loose-envify "^1.1.0" + object-assign "^4.1.1" + prop-types "^15.6.2" + scheduler "^0.13.1" + react-dom@^16.5.2: version "16.5.2" resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-16.5.2.tgz#b69ee47aa20bab5327b2b9d7c1fe2a30f2cfa9d7" @@ -13803,6 +13812,15 @@ react@^16.3.2: object-assign "^4.1.1" prop-types "^15.6.0" +react@^16.4.2: + version "16.8.1" + resolved "https://registry.npmjs.org/react/-/react-16.8.1.tgz#ae11831f6cb2a05d58603a976afc8a558e852c4a" + dependencies: + loose-envify "^1.1.0" + object-assign "^4.1.1" + prop-types "^15.6.2" + scheduler "^0.13.1" + react@^16.5.2: version "16.5.2" resolved "https://registry.yarnpkg.com/react/-/react-16.5.2.tgz#19f6b444ed139baa45609eee6dc3d318b3895d42" @@ -14674,6 +14692,13 @@ schedule@^0.5.0: dependencies: object-assign "^4.1.1" +scheduler@^0.13.1: + version "0.13.1" + resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.13.1.tgz#1a217df1bfaabaf4f1b92a9127d5d732d85a9591" + dependencies: + loose-envify "^1.1.0" + object-assign "^4.1.1" + schema-utils@^0.4.4: version "0.4.7" resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.7.tgz#ba74f597d2be2ea880131746ee17d0a093c68187" From e8b60ab2926e68033100cd4253630138127490dd Mon Sep 17 00:00:00 2001 From: fragosti Date: Wed, 13 Feb 2019 13:41:20 -0800 Subject: [PATCH 12/37] Use remainingFillableAmount instead of maker/takerAssetAmount for amount field for Radar orders --- packages/pipeline/src/parsers/radar_orders/index.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/pipeline/src/parsers/radar_orders/index.ts b/packages/pipeline/src/parsers/radar_orders/index.ts index 47d704a737..c62a1f3819 100644 --- a/packages/pipeline/src/parsers/radar_orders/index.ts +++ b/packages/pipeline/src/parsers/radar_orders/index.ts @@ -84,13 +84,10 @@ function _toGeneric(radarMarket: RadarMarket, radarOrder: RadarSignedOrder): Gen if (radarMarket.baseTokenDecimals === undefined) { return undefined; } - const rawAmount = - radarOrder.type === RadarOrderType.ASK - ? radarOrder.signedOrder.makerAssetAmount - : radarOrder.signedOrder.takerAssetAmount; return { price: radarOrder.price.toString(), - amount: Web3Wrapper.toUnitAmount(new BigNumber(rawAmount.toString()), radarMarket.baseTokenDecimals).toString(), + // Use the remaining fillable amount + amount: radarOrder.remainingQuoteTokenAmount.toString(), }; } From ab286d851d7690543863f99ded50aec728e5c839 Mon Sep 17 00:00:00 2001 From: askeluv Date: Thu, 14 Feb 2019 15:30:58 +0800 Subject: [PATCH 13/37] Changed Maps to EdpsWrappers --- .../src/data_sources/dex_prices/index.ts | 10 ++++++---- .../pipeline/src/parsers/slippage/index.ts | 10 +++++----- .../pipeline/src/scripts/pull_slippage.ts | 19 +++++++++++++------ .../test/parsers/slippage/index_test.ts | 12 +++++------- 4 files changed, 29 insertions(+), 22 deletions(-) diff --git a/packages/pipeline/src/data_sources/dex_prices/index.ts b/packages/pipeline/src/data_sources/dex_prices/index.ts index afa4db6953..a54edf5ebd 100644 --- a/packages/pipeline/src/data_sources/dex_prices/index.ts +++ b/packages/pipeline/src/data_sources/dex_prices/index.ts @@ -24,14 +24,16 @@ export class EdpsSource { /** * Call Ethereum DEX Price Service API. */ - public async getEdpsAsync(direction: string, symbol: string, amount: number): Promise> { - const edpsUrl = `${EDPS_BASE_URL}/${direction}?symbol=${symbol}&amount=${amount}`; + public async getEdpsAsync(direction: string, symbol: string, amount: number): Promise { + const edpsUrl = `${EDPS_BASE_URL}/${direction}?amount=${amount}&symbol=${symbol}&decimals=`; const resp = await fetchAsync(edpsUrl); const respJson: EdpsResponse = await resp.json(); - const allExchanges = new Map(); + const allExchanges: EdpsWrapper = {}; + // The below unwraps the response so we get 1 single EdpsWrapper object + // instead of a list of singletons for (const entry of respJson) { for (const key of Object.keys(entry)) { - allExchanges.set(key, entry[key]); + allExchanges[key] = entry[key]; } } return allExchanges; diff --git a/packages/pipeline/src/parsers/slippage/index.ts b/packages/pipeline/src/parsers/slippage/index.ts index 7f5904fb3d..38e2e6f4a8 100644 --- a/packages/pipeline/src/parsers/slippage/index.ts +++ b/packages/pipeline/src/parsers/slippage/index.ts @@ -1,6 +1,6 @@ import { BigNumber } from '@0x/utils'; -import { EdpsExchange } from '../../data_sources/dex_prices'; +import { EdpsWrapper } from '../../data_sources/dex_prices'; import { Slippage } from '../../entities'; /** @@ -16,11 +16,11 @@ import { Slippage } from '../../entities'; export function calculateSlippage( usdAmount: number, exchange: string, - buyEdps: Map, - sellEdps: Map, + buyEdps: EdpsWrapper, + sellEdps: EdpsWrapper, ): Slippage { - const b = buyEdps.get(exchange); - const s = sellEdps.get(exchange); + const b = buyEdps[exchange]; + const s = sellEdps[exchange]; const slippage = new Slippage(); if (b && s && b.avgPrice && s.avgPrice) { slippage.observedTimestamp = b.timestamp; diff --git a/packages/pipeline/src/scripts/pull_slippage.ts b/packages/pipeline/src/scripts/pull_slippage.ts index 0a0bb3360b..d344bc7489 100644 --- a/packages/pipeline/src/scripts/pull_slippage.ts +++ b/packages/pipeline/src/scripts/pull_slippage.ts @@ -22,6 +22,7 @@ const USD_AMOUNTS = [10, 100, 1000, 10000]; // TODO: fetch from database const TOKENS = ['BAT', 'DAI', 'FUN', 'MANA', 'OMG', 'REP', 'TUSD', 'ZRX', 'MKR', 'BNB', 'USDC', 'LOOM', 'DNT', 'CVC']; +//const TOKENS = ['BAT', 'DNT', 'CVC', 'MANA']; let connection: Connection; @@ -37,12 +38,18 @@ let connection: Connection; return Promise.all( USD_AMOUNTS.map(async usdAmount => { const amount = usdAmount / usdPrice; - const buyEdps = await edpsSource.getEdpsAsync('buy', symbol, amount); - const sellEdps = await edpsSource.getEdpsAsync('sell', symbol, amount); - return Array.from(buyEdps.keys()).map(exchange => { - const slippage: Slippage = calculateSlippage(usdAmount, exchange, buyEdps, sellEdps); - return slippage; - }); + try { + const buyEdps = await edpsSource.getEdpsAsync('buy', symbol, amount); + const sellEdps = await edpsSource.getEdpsAsync('sell', symbol, amount); + return Object.keys(buyEdps).map(exchange => { + const slippage: Slippage = calculateSlippage(usdAmount, exchange, buyEdps, sellEdps); + return slippage; + }); + } catch (e) { + logUtils.log(`Error getting data for symbol=${symbol}, amount=${amount}`); + logUtils.log(e); + return [new Slippage()]; + } }), ); }), diff --git a/packages/pipeline/test/parsers/slippage/index_test.ts b/packages/pipeline/test/parsers/slippage/index_test.ts index ce205b1698..892b136efd 100644 --- a/packages/pipeline/test/parsers/slippage/index_test.ts +++ b/packages/pipeline/test/parsers/slippage/index_test.ts @@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils'; import * as chai from 'chai'; import 'mocha'; -import { EdpsExchange } from '../../../src/data_sources/dex_prices'; +import { EdpsExchange, EdpsWrapper } from '../../../src/data_sources/dex_prices'; import { Slippage } from '../../../src/entities'; import { calculateSlippage } from '../../../src/parsers/slippage'; import { chaiSetup } from '../../utils/chai_setup'; @@ -22,8 +22,8 @@ describe('slippage', () => { const sellPrice = 9; const expectedSlippage = 0.1; - const buyEdps = new Map(); - const buyOrder: EdpsExchange = { + const buyEdps: EdpsWrapper = {}; + buyEdps[exchange] = { exchangeName: exchange, totalPrice: buyPrice, tokenAmount: amount, @@ -32,10 +32,9 @@ describe('slippage', () => { timestamp: ts, error: '', }; - buyEdps.set(exchange, buyOrder); - const sellEdps = new Map(); - const sellOrder: EdpsExchange = { + const sellEdps: EdpsWrapper = {}; + sellEdps[exchange] = { exchangeName: exchange, totalPrice: sellPrice, tokenAmount: amount, @@ -44,7 +43,6 @@ describe('slippage', () => { timestamp: ts, error: '', }; - sellEdps.set(exchange, sellOrder); const expected = new Slippage(); expected.observedTimestamp = ts; expected.symbol = symbol; From d61a7c360e3dd56d6c83528dc0e33f6b855d7d97 Mon Sep 17 00:00:00 2001 From: askeluv Date: Thu, 14 Feb 2019 17:12:31 +0800 Subject: [PATCH 14/37] Self-hosted EDPS --- packages/pipeline/src/data_sources/dex_prices/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pipeline/src/data_sources/dex_prices/index.ts b/packages/pipeline/src/data_sources/dex_prices/index.ts index a54edf5ebd..3796aaa75b 100644 --- a/packages/pipeline/src/data_sources/dex_prices/index.ts +++ b/packages/pipeline/src/data_sources/dex_prices/index.ts @@ -1,6 +1,6 @@ import { fetchAsync } from '@0x/utils'; -const EDPS_BASE_URL = 'https://ethereum-dex-prices-service.production.airswap.io'; +const EDPS_BASE_URL = 'http://35.185.219.196:1337'; export type EdpsResponse = EdpsWrapper[]; From 0e644da6acf8c417bd51afd8121707091047d11c Mon Sep 17 00:00:00 2001 From: askeluv Date: Thu, 14 Feb 2019 17:39:12 +0800 Subject: [PATCH 15/37] Fixed linting issues --- .../src/data_sources/ohlcv_external/crypto_compare.ts | 5 +++-- packages/pipeline/src/scripts/pull_slippage.ts | 4 +--- packages/pipeline/test/parsers/slippage/index_test.ts | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts index 526fc5f6bb..961e6ff61a 100644 --- a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts +++ b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts @@ -49,6 +49,7 @@ export class CryptoCompareOHLCVSource { public readonly defaultExchange = 'CCCAGG'; public readonly interval = this.intervalBetweenRecords * MAX_PAGE_SIZE; // the hourly API returns data for one interval at a time private readonly _url: string = 'https://min-api.cryptocompare.com/data/histohour?'; + private readonly _priceUrl: string = 'https://min-api.cryptocompare.com/data/price?'; // rate-limit for all API calls through this class instance private readonly _limiter: Bottleneck; @@ -102,8 +103,8 @@ export class CryptoCompareOHLCVSource { } public async getUsdPriceAsync(symbol: string): Promise { - const priceUrl = `https://min-api.cryptocompare.com/data/price?tsyms=USD&fsym=${symbol}`; - const resp = await fetchAsync(priceUrl); + const usdUrl = `${this._priceUrl}tsyms=USD&fsym=${symbol}`; + const resp = await fetchAsync(usdUrl); const respJson: CryptoCompareUsdPrice = await resp.json(); return respJson.USD; } diff --git a/packages/pipeline/src/scripts/pull_slippage.ts b/packages/pipeline/src/scripts/pull_slippage.ts index d344bc7489..bfdd1f7b4f 100644 --- a/packages/pipeline/src/scripts/pull_slippage.ts +++ b/packages/pipeline/src/scripts/pull_slippage.ts @@ -1,10 +1,9 @@ -import * as R from 'ramda'; import { Connection, ConnectionOptions, createConnection } from 'typeorm'; import { logUtils } from '@0x/utils'; -import { CryptoCompareOHLCVSource } from '../data_sources/ohlcv_external/crypto_compare'; import { EdpsSource } from '../data_sources/dex_prices'; +import { CryptoCompareOHLCVSource } from '../data_sources/ohlcv_external/crypto_compare'; import { Slippage } from '../entities'; import * as ormConfig from '../ormconfig'; import { calculateSlippage } from '../parsers/slippage'; @@ -22,7 +21,6 @@ const USD_AMOUNTS = [10, 100, 1000, 10000]; // TODO: fetch from database const TOKENS = ['BAT', 'DAI', 'FUN', 'MANA', 'OMG', 'REP', 'TUSD', 'ZRX', 'MKR', 'BNB', 'USDC', 'LOOM', 'DNT', 'CVC']; -//const TOKENS = ['BAT', 'DNT', 'CVC', 'MANA']; let connection: Connection; diff --git a/packages/pipeline/test/parsers/slippage/index_test.ts b/packages/pipeline/test/parsers/slippage/index_test.ts index 892b136efd..7aeb45caa8 100644 --- a/packages/pipeline/test/parsers/slippage/index_test.ts +++ b/packages/pipeline/test/parsers/slippage/index_test.ts @@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils'; import * as chai from 'chai'; import 'mocha'; -import { EdpsExchange, EdpsWrapper } from '../../../src/data_sources/dex_prices'; +import { EdpsWrapper } from '../../../src/data_sources/dex_prices'; import { Slippage } from '../../../src/entities'; import { calculateSlippage } from '../../../src/parsers/slippage'; import { chaiSetup } from '../../utils/chai_setup'; From 87da6a947d4892d100be6cef28c8920be13b8a65 Mon Sep 17 00:00:00 2001 From: fragosti Date: Thu, 14 Feb 2019 09:22:50 -0800 Subject: [PATCH 16/37] Have migration to maker_address column working --- ...-TokenOrderBookSnapshotsAddMakerAddress.ts | 27 -------------- ...-TokenOrderBookSnapshotsAddMakerAddress.ts | 36 +++++++++++++++++++ packages/pipeline/src/entities/token_order.ts | 6 ++-- .../test/entities/token_order_test.ts | 2 +- 4 files changed, 40 insertions(+), 31 deletions(-) delete mode 100644 packages/pipeline/migrations/1550017139695-TokenOrderBookSnapshotsAddMakerAddress.ts create mode 100644 packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts diff --git a/packages/pipeline/migrations/1550017139695-TokenOrderBookSnapshotsAddMakerAddress.ts b/packages/pipeline/migrations/1550017139695-TokenOrderBookSnapshotsAddMakerAddress.ts deleted file mode 100644 index fb13c91f28..0000000000 --- a/packages/pipeline/migrations/1550017139695-TokenOrderBookSnapshotsAddMakerAddress.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm'; - -const TOKEN_ORDERBOOK_SNAPSHOT_TABLE = 'raw.token_orderbook_snapshots'; -const NEW_COLUMN_NAME = 'maker_address'; - -export class TokenOrderBookSnapshotsAddMakerAddress1550017139695 implements MigrationInterface { - public async up(queryRunner: QueryRunner): Promise { - const snapshotTable = await queryRunner.getTable(TOKEN_ORDERBOOK_SNAPSHOT_TABLE); - if (snapshotTable) { - await queryRunner.addColumn( - TOKEN_ORDERBOOK_SNAPSHOT_TABLE, - new TableColumn({ - name: NEW_COLUMN_NAME, - type: 'varchar', - isPrimary: true, - }), - ); - } - } - - public async down(queryRunner: QueryRunner): Promise { - const snapshotTable = await queryRunner.getTable(TOKEN_ORDERBOOK_SNAPSHOT_TABLE); - if (snapshotTable) { - await queryRunner.dropColumn(snapshotTable, NEW_COLUMN_NAME); - } - } -} diff --git a/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts b/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts new file mode 100644 index 0000000000..01fb7b0cec --- /dev/null +++ b/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts @@ -0,0 +1,36 @@ +import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm'; + +const TOKEN_ORDERBOOK_SNAPSHOT_TABLE = 'raw.token_orderbook_snapshots'; +const NEW_COLUMN_NAME = 'maker_address'; + +export class TokenOrderBookSnapshotsAddMakerAddress1550163069315 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + const snapshotTable = await queryRunner.getTable(TOKEN_ORDERBOOK_SNAPSHOT_TABLE); + if (snapshotTable) { + const newColumn = new TableColumn({ + name: NEW_COLUMN_NAME, + type: 'varchar', + isNullable: true, + }); + await queryRunner.addColumn(TOKEN_ORDERBOOK_SNAPSHOT_TABLE, newColumn); + // backfill null values + await queryRunner.query(` + UPDATE ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE} + SET ${NEW_COLUMN_NAME}='unknown' + WHERE ${NEW_COLUMN_NAME} is NULL; + `); + await queryRunner.query(` + ALTER TABLE ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE} + DROP CONSTRAINT "token_orderbook_snapshots_pkey", + ADD PRIMARY KEY (observed_timestamp, source, order_type, price, base_asset_symbol, quote_asset_symbol, maker_address); + `); + } + } + + public async down(queryRunner: QueryRunner): Promise { + const snapshotTable = await queryRunner.getTable(TOKEN_ORDERBOOK_SNAPSHOT_TABLE); + if (snapshotTable) { + await queryRunner.dropColumn(snapshotTable, NEW_COLUMN_NAME); + } + } +} diff --git a/packages/pipeline/src/entities/token_order.ts b/packages/pipeline/src/entities/token_order.ts index 25baca407d..a1f0006d6e 100644 --- a/packages/pipeline/src/entities/token_order.ts +++ b/packages/pipeline/src/entities/token_order.ts @@ -15,14 +15,14 @@ export class TokenOrderbookSnapshot { public price!: BigNumber; @PrimaryColumn({ name: 'base_asset_symbol' }) public baseAssetSymbol!: string; + @PrimaryColumn({ name: 'quote_asset_symbol' }) + public quoteAssetSymbol!: string; @PrimaryColumn({ type: String, name: 'maker_address', default: 'unknown' }) - public makerAddress!: string | null; + public makerAddress!: string; @Column({ nullable: true, type: String, name: 'base_asset_address' }) public baseAssetAddress!: string | null; @Column({ name: 'base_volume', type: 'numeric', transformer: bigNumberTransformer }) public baseVolume!: BigNumber; - @PrimaryColumn({ name: 'quote_asset_symbol' }) - public quoteAssetSymbol!: string; @Column({ nullable: true, type: String, name: 'quote_asset_address' }) public quoteAssetAddress!: string | null; @Column({ name: 'quote_volume', type: 'numeric', transformer: bigNumberTransformer }) diff --git a/packages/pipeline/test/entities/token_order_test.ts b/packages/pipeline/test/entities/token_order_test.ts index 2101436575..8f2df569b0 100644 --- a/packages/pipeline/test/entities/token_order_test.ts +++ b/packages/pipeline/test/entities/token_order_test.ts @@ -20,7 +20,7 @@ const tokenOrderbookSnapshot: TokenOrderbookSnapshot = { quoteAssetSymbol: 'ABC', quoteAssetAddress: '0x00923b9a074762b93650716333b3e1473a15048e', quoteVolume: new BigNumber(12.3234234), - makerAddress: null, + makerAddress: 'unknown', }; describe('TokenOrderbookSnapshot entity', () => { From 7d68378c3c1c353ec36af95401a4b0df26f206de Mon Sep 17 00:00:00 2001 From: fragosti Date: Thu, 14 Feb 2019 09:40:38 -0800 Subject: [PATCH 17/37] Fix volume calculation bug --- packages/pipeline/src/parsers/radar_orders/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pipeline/src/parsers/radar_orders/index.ts b/packages/pipeline/src/parsers/radar_orders/index.ts index c62a1f3819..4a0a33579e 100644 --- a/packages/pipeline/src/parsers/radar_orders/index.ts +++ b/packages/pipeline/src/parsers/radar_orders/index.ts @@ -87,7 +87,7 @@ function _toGeneric(radarMarket: RadarMarket, radarOrder: RadarSignedOrder): Gen return { price: radarOrder.price.toString(), // Use the remaining fillable amount - amount: radarOrder.remainingQuoteTokenAmount.toString(), + amount: radarOrder.remainingBaseTokenAmount.toString(), }; } From cb5b5167fc6de477d22596073a51ec2a536b1ea0 Mon Sep 17 00:00:00 2001 From: fragosti Date: Thu, 14 Feb 2019 10:40:10 -0800 Subject: [PATCH 18/37] Fix type bug around Radar response --- .../pipeline/src/data_sources/radar/index.ts | 23 +++++++++++++++++-- .../src/parsers/radar_orders/index.ts | 3 +-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/packages/pipeline/src/data_sources/radar/index.ts b/packages/pipeline/src/data_sources/radar/index.ts index d6907a2d24..434d2e4631 100644 --- a/packages/pipeline/src/data_sources/radar/index.ts +++ b/packages/pipeline/src/data_sources/radar/index.ts @@ -1,5 +1,6 @@ +import { orderParsingUtils } from '@0x/order-utils'; import { fetchAsync, logUtils } from '@0x/utils'; -import { RadarBook, RadarMarket } from '@radarrelay/types'; +import { RadarBook, RadarMarket, RadarSignedOrder } from '@radarrelay/types'; const RADAR_BASE_URL = 'https://api.radarrelay.com/v2/'; const ACTIVE_MARKETS_URL = `${RADAR_BASE_URL}/markets`; @@ -10,6 +11,17 @@ export const RADAR_SOURCE = 'radar'; // tslint:disable:prefer-function-over-method // ^ Keep consistency with other sources and help logical organization export class RadarSource { + private static _parseRadarOrderResponse(radarOrderResponse: any): RadarSignedOrder { + return { + ...radarOrderResponse, + ...orderParsingUtils.convertStringsFieldsToBigNumbers(radarOrderResponse, [ + 'remainingBaseTokenAmount', + 'remainingQuoteTokenAmount', + 'price', + ]), + signedOrder: orderParsingUtils.convertOrderStringFieldsToBigNumber(radarOrderResponse.signedOrder), + }; + } /** * Call Radar API to find out which markets they are maintaining orderbooks for. */ @@ -29,6 +41,13 @@ export class RadarSource { logUtils.log(`${marketId}: Retrieving orderbook.`); const marketOrderbookUrl = `${ACTIVE_MARKETS_URL}/${marketId}/book?perPage=${MAX_PER_PAGE}`; const resp = await fetchAsync(marketOrderbookUrl); - return resp.json(); + const jsonResp = await resp.json(); + return { + ...jsonResp, + // tslint:disable-next-line:no-unbound-method + bids: jsonResp.bids.map(RadarSource._parseRadarOrderResponse), + // tslint:disable-next-line:no-unbound-method + asks: jsonResp.asks.map(RadarSource._parseRadarOrderResponse), + }; } } diff --git a/packages/pipeline/src/parsers/radar_orders/index.ts b/packages/pipeline/src/parsers/radar_orders/index.ts index 4a0a33579e..48b16fe3fd 100644 --- a/packages/pipeline/src/parsers/radar_orders/index.ts +++ b/packages/pipeline/src/parsers/radar_orders/index.ts @@ -1,7 +1,6 @@ import { ObjectMap } from '@0x/types'; import { BigNumber } from '@0x/utils'; -import { Web3Wrapper } from '@0x/web3-wrapper'; -import { RadarBook, RadarMarket, RadarOrderType, RadarSignedOrder } from '@radarrelay/types'; +import { RadarBook, RadarMarket, RadarSignedOrder } from '@radarrelay/types'; import * as R from 'ramda'; import { aggregateOrders, GenericRawOrder } from '../utils'; From 58d51e039ceba8449e3c5cba7a45f4547870b264 Mon Sep 17 00:00:00 2001 From: fragosti Date: Thu, 14 Feb 2019 10:45:48 -0800 Subject: [PATCH 19/37] Rename migration to what it is called on prod and qa --- .../1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts b/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts index 01fb7b0cec..38c764b46c 100644 --- a/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts +++ b/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts @@ -21,7 +21,7 @@ export class TokenOrderBookSnapshotsAddMakerAddress1550163069315 implements Migr `); await queryRunner.query(` ALTER TABLE ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE} - DROP CONSTRAINT "token_orderbook_snapshots_pkey", + DROP CONSTRAINT "token_orderbook_snapshots_pkey1", ADD PRIMARY KEY (observed_timestamp, source, order_type, price, base_asset_symbol, quote_asset_symbol, maker_address); `); } From 8f7145f80ae8a7b97aaeaedde05fa826c099f83f Mon Sep 17 00:00:00 2001 From: fragosti Date: Thu, 14 Feb 2019 15:07:09 -0800 Subject: [PATCH 20/37] Add parsing radar order test --- .../pipeline/src/data_sources/radar/index.ts | 6 +- .../test/data_sources/radar/index_test.ts | 77 +++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 packages/pipeline/test/data_sources/radar/index_test.ts diff --git a/packages/pipeline/src/data_sources/radar/index.ts b/packages/pipeline/src/data_sources/radar/index.ts index 434d2e4631..873b0fefe2 100644 --- a/packages/pipeline/src/data_sources/radar/index.ts +++ b/packages/pipeline/src/data_sources/radar/index.ts @@ -11,7 +11,7 @@ export const RADAR_SOURCE = 'radar'; // tslint:disable:prefer-function-over-method // ^ Keep consistency with other sources and help logical organization export class RadarSource { - private static _parseRadarOrderResponse(radarOrderResponse: any): RadarSignedOrder { + public static parseRadarOrderResponse(radarOrderResponse: any): RadarSignedOrder { return { ...radarOrderResponse, ...orderParsingUtils.convertStringsFieldsToBigNumbers(radarOrderResponse, [ @@ -45,9 +45,9 @@ export class RadarSource { return { ...jsonResp, // tslint:disable-next-line:no-unbound-method - bids: jsonResp.bids.map(RadarSource._parseRadarOrderResponse), + bids: jsonResp.bids.map(RadarSource.parseRadarOrderResponse), // tslint:disable-next-line:no-unbound-method - asks: jsonResp.asks.map(RadarSource._parseRadarOrderResponse), + asks: jsonResp.asks.map(RadarSource.parseRadarOrderResponse), }; } } diff --git a/packages/pipeline/test/data_sources/radar/index_test.ts b/packages/pipeline/test/data_sources/radar/index_test.ts new file mode 100644 index 0000000000..9a1eacb164 --- /dev/null +++ b/packages/pipeline/test/data_sources/radar/index_test.ts @@ -0,0 +1,77 @@ +import { BigNumber } from '@0x/utils'; +import { RadarOrderState, RadarOrderType } from '@radarrelay/types'; +import * as chai from 'chai'; +import 'mocha'; +import * as R from 'ramda'; + +import { RadarSource } from '../../../src/data_sources/radar'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +const rawResponse = { + orderHash: '0x60bc235f7887a50801c8fc1fc18fb0625ac5f3962cdc1bd59567a6929db8b2ec', + type: 'BID', + state: 'OPEN', + baseTokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + quoteTokenAddress: '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359', + remainingBaseTokenAmount: '9.079731811797989766', + remainingQuoteTokenAmount: '1099.999999999999999889', + price: '121.14895272244560081697', + createdDate: '2019-02-13 21:35:53', + signedOrder: { + exchangeAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + senderAddress: '0x0000000000000000000000000000000000000000', + makerAddress: '0x56178a0d5f301baf6cf3e1cd53d9863437345bf9', + takerAddress: '0x0000000000000000000000000000000000000000', + makerAssetData: '0xf47261b000000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359', + takerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + feeRecipientAddress: '0xa258b39954cef5cb142fd567a46cddb31a670124', + makerAssetAmount: '1099999999999999999889', + takerAssetAmount: '9079731811797989766', + makerFee: '0', + takerFee: '0', + expirationTimeSeconds: '1550094353', + signature: + '0x1ce161d02ad63fe7308e9cd5e97583a8873331d1b72d90e9f3863d9fcba2518cb91ab2fe7de94e4afb39742acdc820abbff2dc0622c8d3865917fade62f16322ae03', + salt: '1550093753237', + }, +}; + +const parsedResponse = { + orderHash: '0x60bc235f7887a50801c8fc1fc18fb0625ac5f3962cdc1bd59567a6929db8b2ec', + type: 'BID' as RadarOrderType, + state: 'OPEN' as RadarOrderState, + baseTokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + quoteTokenAddress: '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359', + remainingBaseTokenAmount: new BigNumber('9.079731811797989766'), + remainingQuoteTokenAmount: new BigNumber('1099.999999999999999889'), + price: new BigNumber('121.14895272244560081697'), + createdDate: '2019-02-13 21:35:53', + signedOrder: { + exchangeAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + senderAddress: '0x0000000000000000000000000000000000000000', + makerAddress: '0x56178a0d5f301baf6cf3e1cd53d9863437345bf9', + takerAddress: '0x0000000000000000000000000000000000000000', + makerAssetData: '0xf47261b000000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359', + takerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + feeRecipientAddress: '0xa258b39954cef5cb142fd567a46cddb31a670124', + makerAssetAmount: new BigNumber('1099999999999999999889'), + takerAssetAmount: new BigNumber('9079731811797989766'), + makerFee: new BigNumber('0'), + takerFee: new BigNumber('0'), + expirationTimeSeconds: new BigNumber('1550094353'), + signature: + '0x1ce161d02ad63fe7308e9cd5e97583a8873331d1b72d90e9f3863d9fcba2518cb91ab2fe7de94e4afb39742acdc820abbff2dc0622c8d3865917fade62f16322ae03', + salt: new BigNumber('1550093753237'), + }, +}; + +describe('RadarSource', () => { + describe('parseRadarOrderResponse', () => { + it('Correctly parses a Radar orderbook response to a RadarBook', () => { + expect(RadarSource.parseRadarOrderResponse(rawResponse)).deep.equal(parsedResponse); + }); + }); +}); From f33f808dc79672f00b336164274e03d9f837d4e0 Mon Sep 17 00:00:00 2001 From: fragosti Date: Thu, 14 Feb 2019 17:17:35 -0800 Subject: [PATCH 21/37] fix null column constraint issue --- packages/pipeline/src/parsers/ddex_orders/index.ts | 2 ++ packages/pipeline/src/parsers/idex_orders/index.ts | 1 + packages/pipeline/src/parsers/oasis_orders/index.ts | 1 + packages/pipeline/src/parsers/paradex_orders/index.ts | 1 + 4 files changed, 5 insertions(+) diff --git a/packages/pipeline/src/parsers/ddex_orders/index.ts b/packages/pipeline/src/parsers/ddex_orders/index.ts index 562f894ab6..7e9bbb18aa 100644 --- a/packages/pipeline/src/parsers/ddex_orders/index.ts +++ b/packages/pipeline/src/parsers/ddex_orders/index.ts @@ -65,5 +65,7 @@ export function parseDdexOrder( tokenOrder.quoteAssetSymbol = ddexMarket.quoteToken; tokenOrder.quoteAssetAddress = ddexMarket.quoteTokenAddress; tokenOrder.quoteVolume = price.times(amount); + + tokenOrder.makerAddress = 'unknown'; return tokenOrder; } diff --git a/packages/pipeline/src/parsers/idex_orders/index.ts b/packages/pipeline/src/parsers/idex_orders/index.ts index 14b8711958..8620236b89 100644 --- a/packages/pipeline/src/parsers/idex_orders/index.ts +++ b/packages/pipeline/src/parsers/idex_orders/index.ts @@ -65,6 +65,7 @@ export function parseIdexOrder( tokenOrder.price = price; tokenOrder.baseVolume = amount; tokenOrder.quoteVolume = price.times(amount); + tokenOrder.makerAddress = 'unknown'; if (orderType === OrderType.Bid) { tokenOrder.baseAssetSymbol = idexOrderParam.buySymbol; diff --git a/packages/pipeline/src/parsers/oasis_orders/index.ts b/packages/pipeline/src/parsers/oasis_orders/index.ts index b71fb65b90..145a547c0b 100644 --- a/packages/pipeline/src/parsers/oasis_orders/index.ts +++ b/packages/pipeline/src/parsers/oasis_orders/index.ts @@ -67,5 +67,6 @@ export function parseOasisOrder( tokenOrder.quoteAssetSymbol = oasisMarket.quote; tokenOrder.quoteAssetAddress = null; // Oasis doesn't provide address information tokenOrder.quoteVolume = price.times(amount); + tokenOrder.makerAddress = 'unknown'; return tokenOrder; } diff --git a/packages/pipeline/src/parsers/paradex_orders/index.ts b/packages/pipeline/src/parsers/paradex_orders/index.ts index 85990dae4f..5f41d9676c 100644 --- a/packages/pipeline/src/parsers/paradex_orders/index.ts +++ b/packages/pipeline/src/parsers/paradex_orders/index.ts @@ -62,5 +62,6 @@ export function parseParadexOrder( tokenOrder.quoteAssetSymbol = paradexMarket.quoteToken; tokenOrder.quoteAssetAddress = paradexMarket.quoteTokenAddress as string; tokenOrder.quoteVolume = price.times(amount); + tokenOrder.makerAddress = 'unknown'; return tokenOrder; } From 44300dabb036fb4941c35405b82326b23aa56408 Mon Sep 17 00:00:00 2001 From: Jacob Evans Date: Mon, 18 Feb 2019 20:04:55 -0800 Subject: [PATCH 22/37] Fix bug in ledger address selector where provider does not exist. This can happen in browsers without metamask or a injected provider --- .../ts/pages/governance/connect_form.tsx | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/packages/website/ts/pages/governance/connect_form.tsx b/packages/website/ts/pages/governance/connect_form.tsx index be9868900c..54244fb07f 100644 --- a/packages/website/ts/pages/governance/connect_form.tsx +++ b/packages/website/ts/pages/governance/connect_form.tsx @@ -220,20 +220,14 @@ export class ConnectForm extends React.Component { } public async getZrxBalanceAsync(owner: string): Promise { utils.assert(!_.isUndefined(this._contractWrappers), 'ContractWrappers must be instantiated.'); - const injectedProvider = await this._getInjectedProviderIfExistsAsync(); - - if (!_.isUndefined(injectedProvider)) { - const contractAddresses = getContractAddressesForNetworkOrThrow(this.networkId); - const tokenAddress: string = contractAddresses.zrxToken; - try { - const amount = await this._contractWrappers.erc20Token.getBalanceAsync(tokenAddress, owner); - return amount; - } catch (error) { - return ZERO; - } + const contractAddresses = getContractAddressesForNetworkOrThrow(this.networkId); + const tokenAddress: string = contractAddresses.zrxToken; + try { + const amount = await this._contractWrappers.erc20Token.getBalanceAsync(tokenAddress, owner); + return amount; + } catch (error) { + return ZERO; } - - return ZERO; } private async _onConnectWalletClickAsync(): Promise { const shouldUseLedgerProvider = false; From 6e0260385c6b07ff460e629fca21467fdf49567b Mon Sep 17 00:00:00 2001 From: Fred Carlsen Date: Tue, 19 Feb 2019 13:32:25 +0100 Subject: [PATCH 23/37] Added ledger sign notification --- .../ts/pages/governance/ledger_sign_note.tsx | 50 +++++++++++++++++++ .../website/ts/pages/governance/vote_form.tsx | 22 ++++++-- 2 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 packages/website/ts/pages/governance/ledger_sign_note.tsx diff --git a/packages/website/ts/pages/governance/ledger_sign_note.tsx b/packages/website/ts/pages/governance/ledger_sign_note.tsx new file mode 100644 index 0000000000..08d472a7ac --- /dev/null +++ b/packages/website/ts/pages/governance/ledger_sign_note.tsx @@ -0,0 +1,50 @@ +import * as _ from 'lodash'; +import * as React from 'react'; +import styled from 'styled-components'; + +import { colors } from 'ts/style/colors'; + +interface LedgerSignNoteProps { + text?: string; + isVisible: boolean; +} + +export const LedgerSignNote: React.StatelessComponent = ({ + text, + isVisible, +}) => { + return ( + + + {text} + + + ); +}; + +LedgerSignNote.defaultProps = { + isVisible: false, +}; + +const Wrapper = styled.div` + background-color: #7A7A7A; + display: flex; + align-items: center; + padding: 28px 30px; + width: 100%; + position: absolute; + top: 0; + bottom: 0; + left: 0; + right: 0; + justify-content: center; + opacity: ${props => props.isVisible ? 1 : 0}; + visibility: ${props => props.isVisible ? 'visible' : 'hidden'}; +`; + +const Text = styled.p` + color: ${colors.white}; + font-size: 1rem; + line-height: 1; + font-weight: 400; +`; diff --git a/packages/website/ts/pages/governance/vote_form.tsx b/packages/website/ts/pages/governance/vote_form.tsx index d4c5b6bc8b..fc5686f0a4 100644 --- a/packages/website/ts/pages/governance/vote_form.tsx +++ b/packages/website/ts/pages/governance/vote_form.tsx @@ -14,6 +14,7 @@ import styled from 'styled-components'; import { Button } from 'ts/components/button'; import { Input } from 'ts/components/modals/input'; import { Heading, Paragraph } from 'ts/components/text'; +import { LedgerSignNote } from 'ts/pages/governance/ledger_sign_note'; import { PreferenceSelecter } from 'ts/pages/governance/preference_selecter'; import { colors } from 'ts/style/colors'; import { InjectedProvider } from 'ts/types'; @@ -50,6 +51,7 @@ interface State { isWalletConnected: boolean; isSubmitting: boolean; isSuccessful: boolean; + isAwaitingLedgerSignature: boolean; isVoted: boolean; selectedAddress?: string; votePreference?: string; @@ -95,6 +97,7 @@ export class VoteForm extends React.Component { public networkId: number; public state: State = { isWalletConnected: false, + isAwaitingLedgerSignature: false, isSubmitting: false, isSuccessful: false, isVoted: false, @@ -116,7 +119,7 @@ export class VoteForm extends React.Component { super(props); } public render(): React.ReactNode { - const { votePreference, errors, isSuccessful } = this.state; + const { votePreference, errors, isSuccessful, isAwaitingLedgerSignature } = this.state; const { currentBalance, selectedAddress } = this.props; const bigNumberFormat = { decimalSeparator: '.', @@ -185,6 +188,7 @@ export class VoteForm extends React.Component { Back Submit + ); @@ -193,15 +197,19 @@ export class VoteForm extends React.Component { e.preventDefault(); const { zeip, votePreference, comment } = this.state; - const { currentBalance, selectedAddress } = this.props; + const { currentBalance, selectedAddress, isLedger } = this.props; const makerAddress = selectedAddress; + + if (isLedger) { + this.setState({ isAwaitingLedgerSignature: true }); + } + const domainType = [{ name: 'name', type: 'string' }]; const voteType = [ { name: 'preference', type: 'string' }, { name: 'zeip', type: 'uint256' }, { name: 'from', type: 'address' }, ]; - const domainData = { name: '0x Protocol Governance', }; @@ -224,8 +232,8 @@ export class VoteForm extends React.Component { const voteHashHex = `0x${voteHashBuffer.toString('hex')}`; try { const signedVote = await this._signVoteAsync(makerAddress, typedData); - // Store the signed Order - this.setState(prevState => ({ ...prevState, signedVote, voteHash: voteHashHex, isSuccessful: true })); + // Store the signed vote + this.setState(prevState => ({ ...prevState, signedVote, voteHash: voteHashHex, isSuccessful: true, isAwaitingLedgerSignature: false })); const voteDomain = utils.isProduction() ? `https://${configs.DOMAIN_VOTE}` : 'http://localhost:3000'; const voteEndpoint = `${voteDomain}/v1/vote`; @@ -255,6 +263,7 @@ export class VoteForm extends React.Component { signError: errorMessage, }, isSuccessful: false, + isAwaitingLedgerSignature: false, }); } } catch (err) { @@ -266,6 +275,7 @@ export class VoteForm extends React.Component { signError: errorMessage, }, isSuccessful: false, + isAwaitingLedgerSignature: false, }); } }; @@ -340,6 +350,8 @@ const InputRow = styled.div` `; const ButtonRow = styled(InputRow)` + position: relative; + @media (max-width: 768px) { display: flex; flex-direction: column; From 3eac854356bbc6e0df129dfa2adcf9424b98ea4c Mon Sep 17 00:00:00 2001 From: Fred Carlsen Date: Tue, 19 Feb 2019 15:20:47 +0100 Subject: [PATCH 24/37] Revert broken formatting --- .../ts/pages/governance/address_table.tsx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/website/ts/pages/governance/address_table.tsx b/packages/website/ts/pages/governance/address_table.tsx index f861a3aee8..28bfd09687 100644 --- a/packages/website/ts/pages/governance/address_table.tsx +++ b/packages/website/ts/pages/governance/address_table.tsx @@ -68,15 +68,15 @@ export class AddressTable extends React.Component` - background - color: #fff; - border -radius;: 4;px; - Margin-bottom;: $;{props => props.marginBottom || '25px';} - Padding: 10;px; 30;px; - Height: 230;px; - Overflow - y;: auto; + background-color: #fff; + border-radius: 4px; + margin-bottom: ${props => props.marginBottom || '25px'}; + padding: 10px 30px; + height: 230px; + overflow-y: auto; `; -const Table = styled.table`; - border-collapse;: collapse; +const Table = styled.table` + border-collapse: collapse; width: 100%; `; From ab62f50f5ce85e97e826f42624efc2fb5fac929e Mon Sep 17 00:00:00 2001 From: Fred Carlsen Date: Tue, 19 Feb 2019 15:53:07 +0100 Subject: [PATCH 25/37] Hide ledger sign note on error --- .../website/ts/pages/governance/vote_form.tsx | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/packages/website/ts/pages/governance/vote_form.tsx b/packages/website/ts/pages/governance/vote_form.tsx index fc5686f0a4..bafd5572e0 100644 --- a/packages/website/ts/pages/governance/vote_form.tsx +++ b/packages/website/ts/pages/governance/vote_form.tsx @@ -256,29 +256,27 @@ export class VoteForm extends React.Component { } else { const responseBody = await response.json(); const errorMessage = !_.isUndefined(responseBody.reason) ? responseBody.reason : 'Unknown Error'; - this.props.onError - ? this.props.onError(errorMessage) - : this.setState({ - errors: { - signError: errorMessage, - }, - isSuccessful: false, - isAwaitingLedgerSignature: false, - }); + this._handleError(errorMessage); } } catch (err) { - const errorMessage = err.message; - this.props.onError - ? this.props.onError(errorMessage) - : this.setState({ - errors: { - signError: errorMessage, - }, - isSuccessful: false, - isAwaitingLedgerSignature: false, - }); + this._handleError(err.message); } }; + private _handleError(errorMessage: string): void { + const { onError } = this.props; + onError + ? onError(errorMessage) + : this.setState({ + errors: { + signError: errorMessage, + }, + isSuccessful: false, + isAwaitingLedgerSignature: false, + }); + this.setState({ + isAwaitingLedgerSignature: false, + }); + } private async _signVoteAsync(signerAddress: string, typedData: any): Promise { const { provider: providerEngine } = this.props; let signatureHex; From 184e111e39ac840b1879ad030bb16b6e69030f73 Mon Sep 17 00:00:00 2001 From: Jacob Evans Date: Tue, 19 Feb 2019 09:12:29 -0800 Subject: [PATCH 26/37] Fix style for address table --- packages/website/ts/pages/governance/address_table.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/website/ts/pages/governance/address_table.tsx b/packages/website/ts/pages/governance/address_table.tsx index f861a3aee8..e3f88e249f 100644 --- a/packages/website/ts/pages/governance/address_table.tsx +++ b/packages/website/ts/pages/governance/address_table.tsx @@ -70,10 +70,10 @@ export class AddressTable extends React.Component` background - color: #fff; border -radius;: 4;px; - Margin-bottom;: $;{props => props.marginBottom || '25px';} - Padding: 10;px; 30;px; - Height: 230;px; - Overflow - y;: auto; + margin-bottom;: $;{props => props.marginBottom || '25px';} + padding: 10;px; 30;px; + height: 230;px; + overflow - y;: auto; `; const Table = styled.table`; From c16e62b5bffbf9de7443dc27ccfad5c4ebe8ae88 Mon Sep 17 00:00:00 2001 From: Jacob Evans Date: Tue, 19 Feb 2019 11:28:09 -0800 Subject: [PATCH 27/37] Recieved -> Received --- packages/website/ts/pages/governance/modal_vote.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/website/ts/pages/governance/modal_vote.tsx b/packages/website/ts/pages/governance/modal_vote.tsx index b2514c81ad..5c86b38611 100644 --- a/packages/website/ts/pages/governance/modal_vote.tsx +++ b/packages/website/ts/pages/governance/modal_vote.tsx @@ -120,7 +120,7 @@ export class ModalVote extends React.Component { - Vote Recieved! + Vote Received! Your vote will help to decide the future of the protocol. You will be receiving a custom From 5acdab849e1297919e9d8166426c4503ddb83397 Mon Sep 17 00:00:00 2001 From: Jacob Evans Date: Tue, 19 Feb 2019 13:09:05 -0800 Subject: [PATCH 28/37] Prettier --- .../ts/pages/governance/ledger_sign_note.tsx | 15 ++++------- .../website/ts/pages/governance/vote_form.tsx | 25 +++++++++++++------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/packages/website/ts/pages/governance/ledger_sign_note.tsx b/packages/website/ts/pages/governance/ledger_sign_note.tsx index 08d472a7ac..c530704ea7 100644 --- a/packages/website/ts/pages/governance/ledger_sign_note.tsx +++ b/packages/website/ts/pages/governance/ledger_sign_note.tsx @@ -9,15 +9,10 @@ interface LedgerSignNoteProps { isVisible: boolean; } -export const LedgerSignNote: React.StatelessComponent = ({ - text, - isVisible, -}) => { +export const LedgerSignNote: React.StatelessComponent = ({ text, isVisible }) => { return ( - - {text} - + {text} ); }; @@ -27,7 +22,7 @@ LedgerSignNote.defaultProps = { }; const Wrapper = styled.div` - background-color: #7A7A7A; + background-color: #7a7a7a; display: flex; align-items: center; padding: 28px 30px; @@ -38,8 +33,8 @@ const Wrapper = styled.div` left: 0; right: 0; justify-content: center; - opacity: ${props => props.isVisible ? 1 : 0}; - visibility: ${props => props.isVisible ? 'visible' : 'hidden'}; + opacity: ${props => (props.isVisible ? 1 : 0)}; + visibility: ${props => (props.isVisible ? 'visible' : 'hidden')}; `; const Text = styled.p` diff --git a/packages/website/ts/pages/governance/vote_form.tsx b/packages/website/ts/pages/governance/vote_form.tsx index bafd5572e0..d39b2e8628 100644 --- a/packages/website/ts/pages/governance/vote_form.tsx +++ b/packages/website/ts/pages/governance/vote_form.tsx @@ -188,7 +188,10 @@ export class VoteForm extends React.Component { Back Submit - + ); @@ -233,7 +236,13 @@ export class VoteForm extends React.Component { try { const signedVote = await this._signVoteAsync(makerAddress, typedData); // Store the signed vote - this.setState(prevState => ({ ...prevState, signedVote, voteHash: voteHashHex, isSuccessful: true, isAwaitingLedgerSignature: false })); + this.setState(prevState => ({ + ...prevState, + signedVote, + voteHash: voteHashHex, + isSuccessful: true, + isAwaitingLedgerSignature: false, + })); const voteDomain = utils.isProduction() ? `https://${configs.DOMAIN_VOTE}` : 'http://localhost:3000'; const voteEndpoint = `${voteDomain}/v1/vote`; @@ -267,12 +276,12 @@ export class VoteForm extends React.Component { onError ? onError(errorMessage) : this.setState({ - errors: { - signError: errorMessage, - }, - isSuccessful: false, - isAwaitingLedgerSignature: false, - }); + errors: { + signError: errorMessage, + }, + isSuccessful: false, + isAwaitingLedgerSignature: false, + }); this.setState({ isAwaitingLedgerSignature: false, }); From 8b93a2816cf8634ab115f41b8b3a12e87adc1781 Mon Sep 17 00:00:00 2001 From: Jacob Evans Date: Tue, 19 Feb 2019 14:53:18 -0800 Subject: [PATCH 29/37] Show greater than less than when minority has a vote --- .../website/ts/pages/governance/vote_bar.tsx | 25 ++++++++++++++----- .../ts/pages/governance/vote_stats.tsx | 5 ++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/packages/website/ts/pages/governance/vote_bar.tsx b/packages/website/ts/pages/governance/vote_bar.tsx index 797edeb652..a9ec957809 100644 --- a/packages/website/ts/pages/governance/vote_bar.tsx +++ b/packages/website/ts/pages/governance/vote_bar.tsx @@ -1,3 +1,4 @@ +import { BigNumber } from '@0x/utils'; import * as _ from 'lodash'; import * as React from 'react'; import styled from 'styled-components'; @@ -5,7 +6,7 @@ import styled from 'styled-components'; interface VoteBarProps { label: string; color: string; - percentage: string; + percentage: BigNumber; marginBottom?: string; } @@ -14,16 +15,27 @@ interface VoteColumnProps { width: string; } -export const VoteBar: React.StatelessComponent = ({ percentage, color, label, marginBottom }) => { - const percentageLabel = `${percentage}%`; +const buildVotePercentageLabel = (percentage: BigNumber): string => { + let percentageLabel = `${percentage.toFixed(0)}%`; + // When voting is entirely dominated it can result in showing 100% and 0% + // In this case we replace with an indication that there are some votes for + // the minority + if (percentage.isGreaterThan(99) && percentage.isLessThan(100)) { + percentageLabel = `> 99%`; + } else if (percentage.isGreaterThan(0) && percentage.isLessThan(1)) { + percentageLabel = `< 1%`; + } + return percentageLabel; +}; +export const VoteBar: React.StatelessComponent = ({ percentage, color, label, marginBottom }) => { // TODO convert this to use a Container component return ( {label}
- - {percentageLabel} + + {buildVotePercentageLabel(percentage)}
); @@ -33,7 +45,7 @@ const VoteColumn = styled.div` background-color: ${props => props.color}; width: calc(${props => props.width}% - 45px); height: 13px; - margin-right: 10px; + margin-right: 15px; min-width: 10px; `; @@ -55,4 +67,5 @@ const VoteColumnLabel = styled.span` font-size: 1rem; line-height: 1; font-weight: 300; + min-width: 60px; `; diff --git a/packages/website/ts/pages/governance/vote_stats.tsx b/packages/website/ts/pages/governance/vote_stats.tsx index c57a91e0f1..f2c8a38bee 100644 --- a/packages/website/ts/pages/governance/vote_stats.tsx +++ b/packages/website/ts/pages/governance/vote_stats.tsx @@ -11,6 +11,7 @@ import { constants } from 'ts/utils/constants'; interface VoteStatsProps { tally?: TallyInterface; } + export const VoteStats: React.StatelessComponent = ({ tally }) => { const bigNumberFormat = { decimalSeparator: '.', @@ -43,8 +44,8 @@ export const VoteStats: React.StatelessComponent = ({ tally }) = Results - - + + ({totalBalanceString} ZRX total vote) ); From 379e828fc11d9fa908b273d55b24bee8886fb11f Mon Sep 17 00:00:00 2001 From: askeluv Date: Wed, 20 Feb 2019 08:55:21 +0800 Subject: [PATCH 30/37] Moved calculateSlippage from parsers to transformers --- packages/pipeline/src/scripts/pull_slippage.ts | 2 +- .../pipeline/src/{parsers => transformers}/slippage/index.ts | 0 .../test/{parsers => transformers}/slippage/index_test.ts | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename packages/pipeline/src/{parsers => transformers}/slippage/index.ts (100%) rename packages/pipeline/test/{parsers => transformers}/slippage/index_test.ts (96%) diff --git a/packages/pipeline/src/scripts/pull_slippage.ts b/packages/pipeline/src/scripts/pull_slippage.ts index bfdd1f7b4f..6d308ee34a 100644 --- a/packages/pipeline/src/scripts/pull_slippage.ts +++ b/packages/pipeline/src/scripts/pull_slippage.ts @@ -6,7 +6,7 @@ import { EdpsSource } from '../data_sources/dex_prices'; import { CryptoCompareOHLCVSource } from '../data_sources/ohlcv_external/crypto_compare'; import { Slippage } from '../entities'; import * as ormConfig from '../ormconfig'; -import { calculateSlippage } from '../parsers/slippage'; +import { calculateSlippage } from '../transformers/slippage'; import { handleError } from '../utils'; // Number of orders to save at once. diff --git a/packages/pipeline/src/parsers/slippage/index.ts b/packages/pipeline/src/transformers/slippage/index.ts similarity index 100% rename from packages/pipeline/src/parsers/slippage/index.ts rename to packages/pipeline/src/transformers/slippage/index.ts diff --git a/packages/pipeline/test/parsers/slippage/index_test.ts b/packages/pipeline/test/transformers/slippage/index_test.ts similarity index 96% rename from packages/pipeline/test/parsers/slippage/index_test.ts rename to packages/pipeline/test/transformers/slippage/index_test.ts index 7aeb45caa8..8528051215 100644 --- a/packages/pipeline/test/parsers/slippage/index_test.ts +++ b/packages/pipeline/test/transformers/slippage/index_test.ts @@ -4,7 +4,7 @@ import 'mocha'; import { EdpsWrapper } from '../../../src/data_sources/dex_prices'; import { Slippage } from '../../../src/entities'; -import { calculateSlippage } from '../../../src/parsers/slippage'; +import { calculateSlippage } from '../../../src/transformers/slippage'; import { chaiSetup } from '../../utils/chai_setup'; chaiSetup.configure(); From 0c871b67cad84a73c461eec51c4fb24fecd87293 Mon Sep 17 00:00:00 2001 From: fragosti Date: Tue, 19 Feb 2019 16:58:11 -0800 Subject: [PATCH 31/37] Add parser test for Radar orderbook parsing --- packages/pipeline/test/parsers/ddex_orders/index_test.ts | 2 +- packages/pipeline/test/parsers/idex_orders/index_test.ts | 4 ++-- packages/pipeline/test/parsers/oasis_orders/index_test.ts | 2 +- packages/pipeline/test/parsers/paradex_orders/index_test.ts | 2 +- packages/pipeline/test/parsers/radar_orders/index_test.ts | 0 5 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 packages/pipeline/test/parsers/radar_orders/index_test.ts diff --git a/packages/pipeline/test/parsers/ddex_orders/index_test.ts b/packages/pipeline/test/parsers/ddex_orders/index_test.ts index d6f69e090e..7eb74ed09e 100644 --- a/packages/pipeline/test/parsers/ddex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/ddex_orders/index_test.ts @@ -44,7 +44,7 @@ describe('ddex_orders', () => { expected.baseAssetSymbol = 'DEF'; expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; expected.baseVolume = new BigNumber(10); - + expected.makerAddress = 'unknown'; const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, source, ddexOrder); expect(actual).deep.equal(expected); }); diff --git a/packages/pipeline/test/parsers/idex_orders/index_test.ts b/packages/pipeline/test/parsers/idex_orders/index_test.ts index 48b019732a..0c7c642ce2 100644 --- a/packages/pipeline/test/parsers/idex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/idex_orders/index_test.ts @@ -45,7 +45,7 @@ describe('idex_orders', () => { expected.quoteAssetSymbol = 'DEF'; expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; expected.quoteVolume = new BigNumber(5); - + expected.makerAddress = 'unknown'; const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, source, idexOrder); expect(actual).deep.equal(expected); }); @@ -79,7 +79,7 @@ describe('idex_orders', () => { expected.quoteAssetSymbol = 'DEF'; expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; expected.quoteVolume = new BigNumber(5); - + expected.makerAddress = 'unknown'; const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, source, idexOrder); expect(actual).deep.equal(expected); }); diff --git a/packages/pipeline/test/parsers/oasis_orders/index_test.ts b/packages/pipeline/test/parsers/oasis_orders/index_test.ts index 401fedff87..7ef86ee10c 100644 --- a/packages/pipeline/test/parsers/oasis_orders/index_test.ts +++ b/packages/pipeline/test/parsers/oasis_orders/index_test.ts @@ -41,7 +41,7 @@ describe('oasis_orders', () => { expected.quoteAssetSymbol = 'ABC'; expected.quoteAssetAddress = null; expected.quoteVolume = new BigNumber(5); - + expected.makerAddress = 'unknown'; const actual = parseOasisOrder(oasisMarket, observedTimestamp, orderType, source, oasisOrder); expect(actual).deep.equal(expected); }); diff --git a/packages/pipeline/test/parsers/paradex_orders/index_test.ts b/packages/pipeline/test/parsers/paradex_orders/index_test.ts index c5dd8751b2..10ac1ecdc9 100644 --- a/packages/pipeline/test/parsers/paradex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/paradex_orders/index_test.ts @@ -46,7 +46,7 @@ describe('paradex_orders', () => { expected.quoteAssetSymbol = 'ABC'; expected.quoteAssetAddress = '0x0000000000000000000000000000000000000000'; expected.quoteVolume = new BigNumber(412 * 0.1245); - + expected.makerAddress = 'unknown'; const actual = parseParadexOrder(paradexMarket, observedTimestamp, orderType, source, paradexOrder); expect(actual).deep.equal(expected); }); diff --git a/packages/pipeline/test/parsers/radar_orders/index_test.ts b/packages/pipeline/test/parsers/radar_orders/index_test.ts new file mode 100644 index 0000000000..e69de29bb2 From 12fd9c29f0c843746f0e6e979feee8bf21bde764 Mon Sep 17 00:00:00 2001 From: fragosti Date: Tue, 19 Feb 2019 17:19:12 -0800 Subject: [PATCH 32/37] Add radar_orders parseRadarOrder test --- .../test/parsers/radar_orders/index_test.ts | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/packages/pipeline/test/parsers/radar_orders/index_test.ts b/packages/pipeline/test/parsers/radar_orders/index_test.ts index e69de29bb2..e3004d7c2a 100644 --- a/packages/pipeline/test/parsers/radar_orders/index_test.ts +++ b/packages/pipeline/test/parsers/radar_orders/index_test.ts @@ -0,0 +1,56 @@ +import { BigNumber } from '@0x/utils'; +import { RadarBook, RadarMarket, RadarSignedOrder } from '@radarrelay/types'; +import * as chai from 'chai'; +import 'mocha'; + +import { TokenOrderbookSnapshot as TokenOrder } from '../../../src/entities'; +import { AggregateOrdersByMaker, parseRadarOrder } from '../../../src/parsers/radar_orders'; +import { OrderType } from '../../../src/types'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('radar_orders', () => { + describe('parseRadarOrder', () => { + it('converts radarOrder to TokenOrder entity', () => { + const radarOrder: AggregateOrdersByMaker = { + makerAddress: '0x6eC92694ea172ebC430C30fa31De87620967A082', + price: '0.01', + amount: new BigNumber(10000000000), + }; + const radarMarket = ({ + id: 'WETH-DAI', + displayName: 'WETH/DAI', + baseTokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + quoteTokenAddress: '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359', + baseTokenDecimals: 18, + quoteTokenDecimals: 18, + quoteIncrement: 8, + minOrderSize: new BigNumber('0.00692535'), + maxOrderSize: new BigNumber('1000000000'), + score: 99.66, + // Radar types are defined using an older version of BigNumber, so need to be force cast. + } as any) as RadarMarket; + const observedTimestamp: number = Date.now(); + const orderType: OrderType = OrderType.Bid; + const source: string = 'radar'; + + const expected = new TokenOrder(); + expected.source = 'radar'; + expected.observedTimestamp = observedTimestamp; + expected.orderType = OrderType.Bid; + expected.price = new BigNumber(0.01); + expected.quoteAssetSymbol = 'DAI'; + expected.quoteAssetAddress = '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359'; + expected.quoteVolume = new BigNumber(100000000); + expected.baseAssetSymbol = 'WETH'; + expected.baseAssetAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; + expected.baseVolume = new BigNumber(10000000000); + expected.makerAddress = '0x6eC92694ea172ebC430C30fa31De87620967A082'; + const actual = parseRadarOrder(radarMarket, observedTimestamp, orderType, source, radarOrder); + expect(actual).deep.equal(expected); + }); + }); +}); From 522736e36762070bd3a4fa722fa7fb8c27a1c27a Mon Sep 17 00:00:00 2001 From: Jacob Evans Date: Tue, 19 Feb 2019 17:34:06 -0800 Subject: [PATCH 33/37] Change optional text props to mandatory in LedgerSignNote --- .../website/ts/pages/governance/ledger_sign_note.tsx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/website/ts/pages/governance/ledger_sign_note.tsx b/packages/website/ts/pages/governance/ledger_sign_note.tsx index c530704ea7..3513cebf33 100644 --- a/packages/website/ts/pages/governance/ledger_sign_note.tsx +++ b/packages/website/ts/pages/governance/ledger_sign_note.tsx @@ -4,11 +4,14 @@ import styled from 'styled-components'; import { colors } from 'ts/style/colors'; -interface LedgerSignNoteProps { - text?: string; +interface WrapperProps { isVisible: boolean; } +interface LedgerSignNoteProps extends WrapperProps { + text: string; +} + export const LedgerSignNote: React.StatelessComponent = ({ text, isVisible }) => { return ( @@ -21,7 +24,7 @@ LedgerSignNote.defaultProps = { isVisible: false, }; -const Wrapper = styled.div` +const Wrapper = styled.div` background-color: #7a7a7a; display: flex; align-items: center; From 1232a9a03ddba1b29c0100c2c8d79945c1ac4733 Mon Sep 17 00:00:00 2001 From: "F. Eugene Aumson" Date: Tue, 19 Feb 2019 19:07:42 -0800 Subject: [PATCH 34/37] Integrate one-time dump and API for nonfungible.com (#1603) * Add script for pulling NFT trade data from nonfungible.com * corrections for current state of API * change data model to match data source * change primary key * pull data from initial dump first, then API * pull all supported NFT's, not just cryptokitties * disable problematic data sources * rename function to satisfy linter * Rename table to nonfungible_dot_com_trades * rename parser module to nonfungible_dot_com from non_fungible_dot_com, for consistency * correct mistaken reference to Bloxy * rename NonfungibleDotComTrade to ...TradeResponse * `NftTrade` -> `NonfungibleDotComTrade` * rename files to match prior object renaming * use fetchAsync instead of axios * improve fetchAsync error message: include URL * avoid non-null contraints in API trades too, not just for trades from the one-time dump * disable mythereum publisher --- .../1543540108767-CreateNftTrades.ts | 31 +++ .../data_sources/nonfungible_dot_com/index.ts | 220 ++++++++++++++++++ packages/pipeline/src/entities/index.ts | 1 + .../src/entities/nonfungible_dot_com_trade.ts | 35 +++ packages/pipeline/src/ormconfig.ts | 2 + .../src/parsers/nonfungible_dot_com/index.ts | 42 ++++ .../pull_nonfungible_dot_com_trades.ts | 43 ++++ packages/pipeline/src/utils/index.ts | 15 +- .../pipeline/test/entities/nft_trades_test.ts | 48 ++++ .../parsers/nonfungible_dot_com/index_test.ts | 86 +++++++ 10 files changed, 522 insertions(+), 1 deletion(-) create mode 100644 packages/pipeline/migrations/1543540108767-CreateNftTrades.ts create mode 100644 packages/pipeline/src/data_sources/nonfungible_dot_com/index.ts create mode 100644 packages/pipeline/src/entities/nonfungible_dot_com_trade.ts create mode 100644 packages/pipeline/src/parsers/nonfungible_dot_com/index.ts create mode 100644 packages/pipeline/src/scripts/pull_nonfungible_dot_com_trades.ts create mode 100644 packages/pipeline/test/entities/nft_trades_test.ts create mode 100644 packages/pipeline/test/parsers/nonfungible_dot_com/index_test.ts diff --git a/packages/pipeline/migrations/1543540108767-CreateNftTrades.ts b/packages/pipeline/migrations/1543540108767-CreateNftTrades.ts new file mode 100644 index 0000000000..a35e8aee24 --- /dev/null +++ b/packages/pipeline/migrations/1543540108767-CreateNftTrades.ts @@ -0,0 +1,31 @@ +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; + +const nftTrades = new Table({ + name: 'raw.nonfungible_dot_com_trades', + columns: [ + { name: 'publisher', type: 'varchar', isPrimary: true }, + { name: 'transaction_hash', type: 'varchar', isPrimary: true }, + { name: 'asset_id', type: 'varchar', isPrimary: true }, + { name: 'block_number', type: 'bigint', isPrimary: true }, + { name: 'log_index', type: 'integer', isPrimary: true }, + + { name: 'block_timestamp', type: 'bigint' }, + { name: 'asset_descriptor', type: 'varchar' }, + { name: 'market_address', type: 'varchar(42)' }, + { name: 'total_price', type: 'numeric' }, + { name: 'usd_price', type: 'numeric' }, + { name: 'buyer_address', type: 'varchar(42)' }, + { name: 'seller_address', type: 'varchar(42)' }, + { name: 'meta', type: 'jsonb' }, + ], +}); + +export class CreateNftTrades1543540108767 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable(nftTrades); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable(nftTrades); + } +} diff --git a/packages/pipeline/src/data_sources/nonfungible_dot_com/index.ts b/packages/pipeline/src/data_sources/nonfungible_dot_com/index.ts new file mode 100644 index 0000000000..7ec701ba62 --- /dev/null +++ b/packages/pipeline/src/data_sources/nonfungible_dot_com/index.ts @@ -0,0 +1,220 @@ +import { stringify } from 'querystring'; + +import { logUtils } from '@0x/utils'; + +import { fetchSuccessfullyOrThrowAsync } from '../../utils'; + +// URL to use for getting nft trades from nonfungible.com. +export const NONFUNGIBLE_DOT_COM_URL = 'https://nonfungible.com/api/v1'; +// Number of trades to get at once. This is a hard limit enforced by the API. +const MAX_TRADES_PER_QUERY = 100; + +// Note(albrow): For now this will have to be manually updated by checking +// https://nonfungible.com/ +export const knownPublishers = [ + 'axieinfinity', + // 'cryptokitties', // disabled until we get updated initial dump that isn't truncated + 'cryptopunks', + 'cryptovoxels', + 'decentraland', + 'decentraland_estate', + 'etherbots', + 'etheremon', + 'ethtown', + // 'knownorigin', // disabled because of null characters in data being rejected by postgres + // 'mythereum', // worked at one time, but now seems dead + 'superrare', +]; + +export interface NonfungibleDotComHistoryResponse { + data: NonfungibleDotComTradeResponse[]; +} + +export interface NonfungibleDotComTradeResponse { + _id: string; + transactionHash: string; + blockNumber: number; + logIndex: number; + blockTimestamp: string; + assetId: string; + assetDescriptor: string; + nftAddress: string; + marketAddress: string; + tokenTicker: string; + totalDecimalPrice: number; + totalPrice: string; + usdPrice: number; + currencyTransfer: object; + buyer: string; + seller: string; + meta: object; + image: string; + composedOf: string; + asset_link: string; + seller_address_link: string; + buyer_address_link: string; +} + +/** + * Gets and returns all trades for the given publisher, starting at the given block number. + * Automatically handles pagination. + * @param publisher A valid "publisher" for the nonfungible.com API. (e.g. "cryptokitties") + * @param blockNumberStart The block number to start querying from. + */ +export async function getTradesAsync( + publisher: string, + blockNumberStart: number, +): Promise { + const allTrades: NonfungibleDotComTradeResponse[] = []; + + /** + * due to high data volumes and rate limiting, we procured an initial data + * dump from nonfungible.com. If the requested starting block number is + * contained in that initial dump, then pull relevant trades from there + * first. Later (below) we'll get the more recent trades from the API itself. + */ + + if (blockNumberStart < highestBlockNumbersInIntialDump[publisher]) { + logUtils.log('getting trades from one-time dump'); + // caller needs trades that are in the initial data dump, so get them + // from there, then later go to the API for the rest. + const initialDumpResponse: NonfungibleDotComHistoryResponse = await fetchSuccessfullyOrThrowAsync( + getInitialDumpUrl(publisher), + ); + const initialDumpTrades = initialDumpResponse.data; + for (const initialDumpTrade of initialDumpTrades) { + if (!shouldProcessTrade(initialDumpTrade, allTrades)) { + continue; + } + + ensureNonNull(initialDumpTrade); + + allTrades.push(initialDumpTrade); + } + logUtils.log(`got ${allTrades.length} from one-time dump`); + } + + const fullUrl = getFullUrlForPublisher(publisher); + + /** + * API returns trades in reverse chronological order, so highest block + * numbers first. The `start` query parameter indicates how far back in + * time (in number of trades) the results should start. Here we iterate + * over both start parameter values and block numbers simultaneously. + * Start parameter values count up from zero. Block numbers count down + * until reaching the highest block number in the initial dump. + */ + + const blockNumberStop = Math.max(highestBlockNumbersInIntialDump[publisher] + 1, blockNumberStart); + for ( + let startParam = 0, blockNumber = Number.MAX_SAFE_INTEGER; + blockNumber > blockNumberStop; + startParam += MAX_TRADES_PER_QUERY + ) { + const response = await _getTradesWithOffsetAsync(fullUrl, publisher, startParam); + const tradesFromApi = response.data; + logUtils.log( + `got ${ + tradesFromApi.length + } trades from API. blockNumber=${blockNumber}. blockNumberStop=${blockNumberStop}`, + ); + for (const tradeFromApi of tradesFromApi) { + if (tradeFromApi.blockNumber <= blockNumberStop) { + blockNumber = blockNumberStop; + break; + } + if (!shouldProcessTrade(tradeFromApi, allTrades)) { + continue; + } + ensureNonNull(tradeFromApi); + allTrades.push(tradeFromApi); + blockNumber = tradeFromApi.blockNumber; + } + } + + return allTrades; +} + +function shouldProcessTrade( + trade: NonfungibleDotComTradeResponse, + existingTrades: NonfungibleDotComTradeResponse[], +): boolean { + // check to see if this trade is already in existingTrades + const existingTradeIndex = existingTrades.findIndex( + // HACK! making assumptions about composition of primary key + e => + e.transactionHash === trade.transactionHash && + e.logIndex === trade.logIndex && + e.blockNumber === trade.blockNumber, + ); + if (existingTradeIndex !== -1) { + logUtils.log("we've already captured this trade. deciding whether to use the existing record or this one."); + if (trade.blockNumber > existingTrades[existingTradeIndex].blockNumber) { + logUtils.log('throwing out existing trade'); + existingTrades.splice(existingTradeIndex, 1); + } else { + logUtils.log('letting existing trade stand, and skipping processing of this trade'); + return false; + } + } + return true; +} + +const highestBlockNumbersInIntialDump: { [publisher: string]: number } = { + axieinfinity: 7065913, + cryptokitties: 4658171, + cryptopunks: 7058897, + cryptovoxels: 7060783, + decentraland_estate: 7065181, + decentraland: 6938962, + etherbots: 5204980, + etheremon: 7065370, + ethtown: 7064126, + knownorigin: 7065160, + mythereum: 7065311, + superrare: 7065955, +}; + +async function _getTradesWithOffsetAsync( + url: string, + publisher: string, + offset: number, +): Promise { + const resp: NonfungibleDotComHistoryResponse = await fetchSuccessfullyOrThrowAsync( + `${url}?${stringify({ + publisher, + start: offset, + length: MAX_TRADES_PER_QUERY, + })}`, + ); + return resp; +} + +function getFullUrlForPublisher(publisher: string): string { + return `${NONFUNGIBLE_DOT_COM_URL}/market/${publisher}/history`; +} + +function getInitialDumpUrl(publisher: string): string { + return `https://nonfungible-dot-com-one-time-data-dump.s3.amazonaws.com/sales_summary_${publisher}.json`; +} + +function ensureNonNull(trade: NonfungibleDotComTradeResponse): void { + // these fields need to be set in order to avoid non-null + // constraint exceptions upon database insertion. + if (trade.logIndex === undefined) { + // for cryptopunks + trade.logIndex = 0; + } + if (trade.assetDescriptor === undefined) { + // for cryptopunks + trade.assetDescriptor = ''; + } + if (trade.meta === undefined) { + // for cryptopunks + trade.meta = {}; + } + if (trade.marketAddress === null) { + // for decentraland_estate + trade.marketAddress = ''; + } +} diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts index 27c153c079..e686216c55 100644 --- a/packages/pipeline/src/entities/index.ts +++ b/packages/pipeline/src/entities/index.ts @@ -7,6 +7,7 @@ export { DexTrade } from './dex_trade'; export { ExchangeCancelEvent } from './exchange_cancel_event'; export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event'; export { ExchangeFillEvent } from './exchange_fill_event'; +export { NonfungibleDotComTrade } from './nonfungible_dot_com_trade'; export { OHLCVExternal } from './ohlcv_external'; export { Relayer } from './relayer'; export { SraOrder } from './sra_order'; diff --git a/packages/pipeline/src/entities/nonfungible_dot_com_trade.ts b/packages/pipeline/src/entities/nonfungible_dot_com_trade.ts new file mode 100644 index 0000000000..514edafcb2 --- /dev/null +++ b/packages/pipeline/src/entities/nonfungible_dot_com_trade.ts @@ -0,0 +1,35 @@ +import { BigNumber } from '@0x/utils'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'nonfungible_dot_com_trades', schema: 'raw' }) +export class NonfungibleDotComTrade { + @PrimaryColumn({ name: 'transaction_hash' }) + public transactionHash!: string; + @PrimaryColumn({ name: 'publisher' }) + public publisher!: string; + @PrimaryColumn({ name: 'block_number', type: 'bigint', transformer: numberToBigIntTransformer }) + public blockNumber!: number; + @PrimaryColumn({ name: 'log_index' }) + public logIndex!: number; + @PrimaryColumn({ name: 'asset_id' }) + public assetId!: string; + + @Column({ name: 'block_timestamp', type: 'bigint', transformer: numberToBigIntTransformer }) + public blockTimestamp!: number; + @Column({ name: 'asset_descriptor' }) + public assetDescriptor!: string; + @Column({ name: 'market_address' }) + public marketAddress!: string; + @Column({ name: 'total_price', type: 'numeric', transformer: bigNumberTransformer }) + public totalPrice!: BigNumber; + @Column({ name: 'usd_price', type: 'numeric', transformer: bigNumberTransformer }) + public usdPrice!: BigNumber; + @Column({ name: 'buyer_address' }) + public buyerAddress!: string; + @Column({ name: 'seller_address' }) + public sellerAddress!: string; + @Column({ type: 'jsonb' }) + public meta!: object; +} diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 2700714cdd..149e7a3ded 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -12,6 +12,7 @@ import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, + NonfungibleDotComTrade, OHLCVExternal, Relayer, SraOrder, @@ -33,6 +34,7 @@ const entities = [ ExchangeCancelUpToEvent, ExchangeFillEvent, ERC20ApprovalEvent, + NonfungibleDotComTrade, OHLCVExternal, Relayer, SraOrder, diff --git a/packages/pipeline/src/parsers/nonfungible_dot_com/index.ts b/packages/pipeline/src/parsers/nonfungible_dot_com/index.ts new file mode 100644 index 0000000000..48daa3d7fd --- /dev/null +++ b/packages/pipeline/src/parsers/nonfungible_dot_com/index.ts @@ -0,0 +1,42 @@ +import { BigNumber } from '@0x/utils'; +import * as R from 'ramda'; + +import { NonfungibleDotComTradeResponse } from '../../data_sources/nonfungible_dot_com'; +import { NonfungibleDotComTrade } from '../../entities'; + +/** + * Parses a raw trades from the nonfungible.com API and returns an array of + * NonfungibleDotComTrade entities. + * @param rawTrades A raw order response from an SRA endpoint. + */ +export function parseNonFungibleDotComTrades( + rawTrades: NonfungibleDotComTradeResponse[], + publisher: string, +): NonfungibleDotComTrade[] { + return R.map(_parseNonFungibleDotComTrade.bind(null, publisher), rawTrades); +} + +/** + * Converts a single trade from nonfungible.com into an NonfungibleDotComTrade entity. + * @param rawTrade A single trade from the response from the nonfungible.com API. + */ +export function _parseNonFungibleDotComTrade( + publisher: string, + rawTrade: NonfungibleDotComTradeResponse, +): NonfungibleDotComTrade { + const nonfungibleDotComTrade = new NonfungibleDotComTrade(); + nonfungibleDotComTrade.assetDescriptor = rawTrade.assetDescriptor; + nonfungibleDotComTrade.assetId = rawTrade.assetId; + nonfungibleDotComTrade.blockNumber = rawTrade.blockNumber; + nonfungibleDotComTrade.blockTimestamp = new Date(rawTrade.blockTimestamp).getTime(); + nonfungibleDotComTrade.buyerAddress = rawTrade.buyer; + nonfungibleDotComTrade.logIndex = rawTrade.logIndex; + nonfungibleDotComTrade.marketAddress = rawTrade.marketAddress; + nonfungibleDotComTrade.meta = rawTrade.meta; + nonfungibleDotComTrade.sellerAddress = rawTrade.seller; + nonfungibleDotComTrade.totalPrice = new BigNumber(rawTrade.totalPrice); + nonfungibleDotComTrade.transactionHash = rawTrade.transactionHash; + nonfungibleDotComTrade.usdPrice = new BigNumber(rawTrade.usdPrice); + nonfungibleDotComTrade.publisher = publisher; + return nonfungibleDotComTrade; +} diff --git a/packages/pipeline/src/scripts/pull_nonfungible_dot_com_trades.ts b/packages/pipeline/src/scripts/pull_nonfungible_dot_com_trades.ts new file mode 100644 index 0000000000..8d563400cd --- /dev/null +++ b/packages/pipeline/src/scripts/pull_nonfungible_dot_com_trades.ts @@ -0,0 +1,43 @@ +// tslint:disable:no-console +import 'reflect-metadata'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { getTradesAsync, knownPublishers } from '../data_sources/nonfungible_dot_com'; +import { NonfungibleDotComTrade } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseNonFungibleDotComTrades } from '../parsers/nonfungible_dot_com'; +import { handleError } from '../utils'; + +// Number of trades to save at once. +const BATCH_SAVE_SIZE = 1000; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + await getAndSaveTradesAsync(); + process.exit(0); +})().catch(handleError); + +async function getAndSaveTradesAsync(): Promise { + const tradesRepository = connection.getRepository(NonfungibleDotComTrade); + + for (const publisher of knownPublishers) { + console.log(`Getting latest trades for NFT ${publisher}...`); + const tradeWithHighestBlockNumber = await tradesRepository + .createQueryBuilder('nonfungible_dot_com_trades') + .where('nonfungible_dot_com_trades.publisher = :publisher', { publisher }) + .orderBy({ 'nonfungible_dot_com_trades.block_number': 'DESC' }) + .getOne(); + const highestExistingBlockNumber = + tradeWithHighestBlockNumber === undefined ? 0 : tradeWithHighestBlockNumber.blockNumber; + console.log(`Highest block number in existing trades: ${highestExistingBlockNumber}`); + const rawTrades = await getTradesAsync(publisher, highestExistingBlockNumber); + console.log(`Parsing ${rawTrades.length} trades...`); + const trades = parseNonFungibleDotComTrades(rawTrades, publisher); + console.log(`Saving ${rawTrades.length} trades...`); + await tradesRepository.save(trades, { chunk: Math.ceil(trades.length / BATCH_SAVE_SIZE) }); + } + const newTotalTrades = await tradesRepository.count(); + console.log(`Done saving trades. There are now ${newTotalTrades} total NFT trades.`); +} diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts index 094c0178e6..0342481e0e 100644 --- a/packages/pipeline/src/utils/index.ts +++ b/packages/pipeline/src/utils/index.ts @@ -1,4 +1,4 @@ -import { BigNumber } from '@0x/utils'; +import { BigNumber, fetchAsync } from '@0x/utils'; export * from './transformers'; export * from './constants'; @@ -51,3 +51,16 @@ export function handleError(e: any): void { } process.exit(1); } + +/** + * Does fetchAsync(), and checks the status code, throwing if it doesn't indicate success. + */ +export async function fetchSuccessfullyOrThrowAsync(url: string): Promise { + const response = await fetchAsync(url); + if (!response.ok) { + throw new Error( + `Failed to fetch URL ${url}. Unsuccessful HTTP status code (${response.status}): ${response.statusText}`, + ); + } + return response.json(); +} diff --git a/packages/pipeline/test/entities/nft_trades_test.ts b/packages/pipeline/test/entities/nft_trades_test.ts new file mode 100644 index 0000000000..01571e8f71 --- /dev/null +++ b/packages/pipeline/test/entities/nft_trades_test.ts @@ -0,0 +1,48 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; +import 'reflect-metadata'; + +import { NonfungibleDotComTrade } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const baseTrade: NonfungibleDotComTrade = { + assetDescriptor: 'Kitty #1002', + assetId: '1002', + blockNumber: 4608542, + blockTimestamp: 1543544083704, + buyerAddress: '0x316c55d1895a085c4b39a98ecb563f509301aaf7', + logIndex: 28, + marketAddress: '0xb1690C08E213a35Ed9bAb7B318DE14420FB57d8C', + meta: { + cattribute_body: 'munchkin', + cattribute_coloreyes: 'mintgreen', + cattribute_colorprimary: 'orangesoda', + cattribute_colorsecondary: 'coffee', + cattribute_colortertiary: 'kittencream', + cattribute_eyes: 'thicccbrowz', + cattribute_mouth: 'soserious', + cattribute_pattern: 'totesbasic', + generation: '0', + is_exclusive: false, + is_fancy: false, + }, + sellerAddress: '0xba52c75764d6f594735dc735be7f1830cdf58ddf', + totalPrice: new BigNumber('9751388888888889'), + transactionHash: '0x468168419be7e442d5ff32d264fab24087b744bc2e37fdbac7024e1e74f4c6c8', + usdPrice: new BigNumber('3.71957'), + publisher: 'cryptokitties', +}; + +// tslint:disable:custom-no-magic-numbers +describe('NonfungibleDotComTrade entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const tradesRepository = connection.getRepository(NonfungibleDotComTrade); + await testSaveAndFindEntityAsync(tradesRepository, baseTrade); + }); +}); diff --git a/packages/pipeline/test/parsers/nonfungible_dot_com/index_test.ts b/packages/pipeline/test/parsers/nonfungible_dot_com/index_test.ts new file mode 100644 index 0000000000..f7929a5460 --- /dev/null +++ b/packages/pipeline/test/parsers/nonfungible_dot_com/index_test.ts @@ -0,0 +1,86 @@ +// tslint:disable:custom-no-magic-numbers +import { BigNumber } from '@0x/utils'; +import * as chai from 'chai'; +import 'mocha'; + +import { NonfungibleDotComTradeResponse } from '../../../src/data_sources/nonfungible_dot_com'; +import { NonfungibleDotComTrade } from '../../../src/entities'; +import { _parseNonFungibleDotComTrade } from '../../../src/parsers/nonfungible_dot_com'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +const input: NonfungibleDotComTradeResponse = { + _id: '5b4cd04244abdb5ac3a8063f', + assetDescriptor: 'Kitty #1002', + assetId: '1002', + blockNumber: 4608542, + blockTimestamp: '2017-11-23T18:50:19.000Z', + buyer: '0x316c55d1895a085c4b39a98ecb563f509301aaf7', + logIndex: 28, + nftAddress: '0xb1690C08E213a35Ed9bAb7B318DE14420FB57d8C', + marketAddress: '0xb1690C08E213a35Ed9bAb7B318DE14420FB57d8C', + tokenTicker: 'eth', + meta: { + cattribute_body: 'munchkin', + cattribute_coloreyes: 'mintgreen', + cattribute_colorprimary: 'orangesoda', + cattribute_colorsecondary: 'coffee', + cattribute_colortertiary: 'kittencream', + cattribute_eyes: 'thicccbrowz', + cattribute_mouth: 'soserious', + cattribute_pattern: 'totesbasic', + generation: '0', + is_exclusive: false, + is_fancy: false, + }, + seller: '0xba52c75764d6f594735dc735be7f1830cdf58ddf', + totalDecimalPrice: 0.00975138888888889, + totalPrice: '9751388888888889', + transactionHash: '0x468168419be7e442d5ff32d264fab24087b744bc2e37fdbac7024e1e74f4c6c8', + usdPrice: 3.71957, + currencyTransfer: {}, + image: '', + composedOf: '', + asset_link: '', + seller_address_link: '', + buyer_address_link: '', +}; + +const expected: NonfungibleDotComTrade = { + assetDescriptor: 'Kitty #1002', + assetId: '1002', + blockNumber: 4608542, + blockTimestamp: 1511463019000, + buyerAddress: '0x316c55d1895a085c4b39a98ecb563f509301aaf7', + logIndex: 28, + marketAddress: '0xb1690C08E213a35Ed9bAb7B318DE14420FB57d8C', + meta: { + cattribute_body: 'munchkin', + cattribute_coloreyes: 'mintgreen', + cattribute_colorprimary: 'orangesoda', + cattribute_colorsecondary: 'coffee', + cattribute_colortertiary: 'kittencream', + cattribute_eyes: 'thicccbrowz', + cattribute_mouth: 'soserious', + cattribute_pattern: 'totesbasic', + generation: '0', + is_exclusive: false, + is_fancy: false, + }, + sellerAddress: '0xba52c75764d6f594735dc735be7f1830cdf58ddf', + totalPrice: new BigNumber('9751388888888889'), + transactionHash: '0x468168419be7e442d5ff32d264fab24087b744bc2e37fdbac7024e1e74f4c6c8', + usdPrice: new BigNumber('3.71957'), + publisher: 'cryptokitties', +}; + +describe('nonfungible.com', () => { + describe('_parseNonFungibleDotComTrade', () => { + it(`converts NonfungibleDotComTradeResponse to NonfungibleDotComTrade entity`, () => { + const actual = _parseNonFungibleDotComTrade(expected.publisher, input); + expect(actual).deep.equal(expected); + }); + }); +}); From bc7e62db9f4dd3f70d6f8cbb9cdecd01a5b4d278 Mon Sep 17 00:00:00 2001 From: fragosti Date: Wed, 20 Feb 2019 11:40:49 -0800 Subject: [PATCH 35/37] Throw error if cannot find error in table --- .../1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts b/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts index 38c764b46c..1d3b3c3ab5 100644 --- a/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts +++ b/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts @@ -24,6 +24,8 @@ export class TokenOrderBookSnapshotsAddMakerAddress1550163069315 implements Migr DROP CONSTRAINT "token_orderbook_snapshots_pkey1", ADD PRIMARY KEY (observed_timestamp, source, order_type, price, base_asset_symbol, quote_asset_symbol, maker_address); `); + } else { + throw new Error(`Could not find table with name ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE}`); } } @@ -31,6 +33,8 @@ export class TokenOrderBookSnapshotsAddMakerAddress1550163069315 implements Migr const snapshotTable = await queryRunner.getTable(TOKEN_ORDERBOOK_SNAPSHOT_TABLE); if (snapshotTable) { await queryRunner.dropColumn(snapshotTable, NEW_COLUMN_NAME); + } else { + throw new Error(`Could not find table with name ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE}`); } } } From ef5195db901a5ba2546c98f08d26200c8cb0d5b6 Mon Sep 17 00:00:00 2001 From: fragosti Date: Wed, 20 Feb 2019 11:43:32 -0800 Subject: [PATCH 36/37] Fix marquee typo --- packages/pipeline/src/parsers/ddex_orders/index.ts | 2 +- packages/pipeline/src/parsers/idex_orders/index.ts | 2 +- packages/pipeline/src/parsers/oasis_orders/index.ts | 2 +- packages/pipeline/src/parsers/paradex_orders/index.ts | 2 +- packages/pipeline/src/parsers/radar_orders/index.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/pipeline/src/parsers/ddex_orders/index.ts b/packages/pipeline/src/parsers/ddex_orders/index.ts index 7e9bbb18aa..8fe3c32bfd 100644 --- a/packages/pipeline/src/parsers/ddex_orders/index.ts +++ b/packages/pipeline/src/parsers/ddex_orders/index.ts @@ -7,7 +7,7 @@ import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; /** - * Marque function of this file. + * Marquee function of this file. * 1) Takes in orders from an orderbook, * other information attached. * @param ddexOrderbook A raw orderbook that we pull from the Ddex API. diff --git a/packages/pipeline/src/parsers/idex_orders/index.ts b/packages/pipeline/src/parsers/idex_orders/index.ts index 8620236b89..da66d7c13d 100644 --- a/packages/pipeline/src/parsers/idex_orders/index.ts +++ b/packages/pipeline/src/parsers/idex_orders/index.ts @@ -7,7 +7,7 @@ import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; /** - * Marque function of this file. + * Marquee function of this file. * 1) Takes in orders from an orderbook, * 2) Aggregates them by price point, * 3) Parses them into entities which are then saved into the database. diff --git a/packages/pipeline/src/parsers/oasis_orders/index.ts b/packages/pipeline/src/parsers/oasis_orders/index.ts index 145a547c0b..bb3ca97fa6 100644 --- a/packages/pipeline/src/parsers/oasis_orders/index.ts +++ b/packages/pipeline/src/parsers/oasis_orders/index.ts @@ -8,7 +8,7 @@ import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; /** - * Marque function of this file. + * Marquee function of this file. * 1) Takes in orders from an orderbook, * 2) Aggregates them according to price point, * 3) Builds TokenOrder entity with other information attached. diff --git a/packages/pipeline/src/parsers/paradex_orders/index.ts b/packages/pipeline/src/parsers/paradex_orders/index.ts index 5f41d9676c..20e442d5a9 100644 --- a/packages/pipeline/src/parsers/paradex_orders/index.ts +++ b/packages/pipeline/src/parsers/paradex_orders/index.ts @@ -5,7 +5,7 @@ import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; /** - * Marque function of this file. + * Marquee function of this file. * 1) Takes in orders from an orderbook (orders are already aggregated by price point), * 2) For each aggregated order, forms a TokenOrder entity with market data and * other information attached. diff --git a/packages/pipeline/src/parsers/radar_orders/index.ts b/packages/pipeline/src/parsers/radar_orders/index.ts index 48b16fe3fd..20472766d1 100644 --- a/packages/pipeline/src/parsers/radar_orders/index.ts +++ b/packages/pipeline/src/parsers/radar_orders/index.ts @@ -15,7 +15,7 @@ export interface AggregateOrdersByMaker { } /** - * Marque function of this file. + * Marquee function of this file. * 1) Takes in orders from an orderbook, * other information attached. * @param radarOrderbook A raw orderbook that we pull from the radar API. From 4952cd5afb0505e5b3f73ba373372cf941c04e19 Mon Sep 17 00:00:00 2001 From: fragosti Date: Wed, 20 Feb 2019 13:45:14 -0800 Subject: [PATCH 37/37] Factor out redundant source param in parse order functions --- .../pipeline/src/parsers/ddex_orders/index.ts | 15 ++++----------- .../pipeline/src/parsers/idex_orders/index.ts | 17 +++++------------ .../pipeline/src/parsers/oasis_orders/index.ts | 11 ++++------- .../src/parsers/paradex_orders/index.ts | 12 ++++-------- .../pipeline/src/parsers/radar_orders/index.ts | 11 ++++------- .../scripts/pull_ddex_orderbook_snapshots.ts | 4 ++-- .../scripts/pull_idex_orderbook_snapshots.ts | 4 ++-- .../scripts/pull_oasis_orderbook_snapshots.ts | 4 ++-- .../scripts/pull_paradex_orderbook_snapshots.ts | 3 +-- .../scripts/pull_radar_orderbook_snapshots.ts | 4 ++-- .../test/data_sources/radar/index_test.ts | 1 - .../test/parsers/ddex_orders/index_test.ts | 3 +-- .../test/parsers/idex_orders/index_test.ts | 6 ++---- .../test/parsers/oasis_orders/index_test.ts | 3 +-- .../test/parsers/paradex_orders/index_test.ts | 3 +-- .../test/parsers/radar_orders/index_test.ts | 5 ++--- 16 files changed, 37 insertions(+), 69 deletions(-) diff --git a/packages/pipeline/src/parsers/ddex_orders/index.ts b/packages/pipeline/src/parsers/ddex_orders/index.ts index 8fe3c32bfd..6c98c32254 100644 --- a/packages/pipeline/src/parsers/ddex_orders/index.ts +++ b/packages/pipeline/src/parsers/ddex_orders/index.ts @@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils'; import { aggregateOrders } from '../utils'; -import { DdexMarket, DdexOrderbook } from '../../data_sources/ddex'; +import { DDEX_SOURCE, DdexMarket, DdexOrderbook } from '../../data_sources/ddex'; import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; @@ -13,22 +13,16 @@ import { OrderType } from '../../types'; * @param ddexOrderbook A raw orderbook that we pull from the Ddex API. * @param ddexMarket An object containing market data also directly from the API. * @param observedTimestamp Time at which the orders for the market were pulled. - * @param source The exchange where these orders are placed. In this case 'ddex'. */ export function parseDdexOrders( ddexOrderbook: DdexOrderbook, ddexMarket: DdexMarket, observedTimestamp: number, - source: string, ): TokenOrder[] { const aggregatedBids = aggregateOrders(ddexOrderbook.bids); const aggregatedAsks = aggregateOrders(ddexOrderbook.asks); - const parsedBids = aggregatedBids.map(order => - parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Bid, source, order), - ); - const parsedAsks = aggregatedAsks.map(order => - parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Ask, source, order), - ); + const parsedBids = aggregatedBids.map(order => parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Bid, order)); + const parsedAsks = aggregatedAsks.map(order => parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Ask, order)); return parsedBids.concat(parsedAsks); } @@ -46,14 +40,13 @@ export function parseDdexOrder( ddexMarket: DdexMarket, observedTimestamp: number, orderType: OrderType, - source: string, ddexOrder: [string, BigNumber], ): TokenOrder { const tokenOrder = new TokenOrder(); const price = new BigNumber(ddexOrder[0]); const amount = ddexOrder[1]; - tokenOrder.source = source; + tokenOrder.source = DDEX_SOURCE; tokenOrder.observedTimestamp = observedTimestamp; tokenOrder.orderType = orderType; tokenOrder.price = price; diff --git a/packages/pipeline/src/parsers/idex_orders/index.ts b/packages/pipeline/src/parsers/idex_orders/index.ts index da66d7c13d..b151fda59d 100644 --- a/packages/pipeline/src/parsers/idex_orders/index.ts +++ b/packages/pipeline/src/parsers/idex_orders/index.ts @@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils'; import { aggregateOrders } from '../utils'; -import { IdexOrderbook, IdexOrderParam } from '../../data_sources/idex'; +import { IDEX_SOURCE, IdexOrderbook, IdexOrderParam } from '../../data_sources/idex'; import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; @@ -13,17 +13,14 @@ import { OrderType } from '../../types'; * 3) Parses them into entities which are then saved into the database. * @param idexOrderbook raw orderbook that we pull from the Idex API. * @param observedTimestamp Time at which the orders for the market were pulled. - * @param source The exchange where these orders are placed. In this case 'idex'. */ -export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: number, source: string): TokenOrder[] { +export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: number): TokenOrder[] { const aggregatedBids = aggregateOrders(idexOrderbook.bids); // Any of the bid orders' params will work const idexBidOrder = idexOrderbook.bids[0]; const parsedBids = aggregatedBids.length > 0 - ? aggregatedBids.map(order => - parseIdexOrder(idexBidOrder.params, observedTimestamp, OrderType.Bid, source, order), - ) + ? aggregatedBids.map(order => parseIdexOrder(idexBidOrder.params, observedTimestamp, OrderType.Bid, order)) : []; const aggregatedAsks = aggregateOrders(idexOrderbook.asks); @@ -31,9 +28,7 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: const idexAskOrder = idexOrderbook.asks[0]; const parsedAsks = aggregatedAsks.length > 0 - ? aggregatedAsks.map(order => - parseIdexOrder(idexAskOrder.params, observedTimestamp, OrderType.Ask, source, order), - ) + ? aggregatedAsks.map(order => parseIdexOrder(idexAskOrder.params, observedTimestamp, OrderType.Ask, order)) : []; return parsedBids.concat(parsedAsks); } @@ -45,21 +40,19 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: * trades have been placed. * @param observedTimestamp The time when the API response returned back to us. * @param orderType 'bid' or 'ask' enum. - * @param source Exchange where these orders were placed. * @param idexOrder A tuple which we will convert to volume-basis. */ export function parseIdexOrder( idexOrderParam: IdexOrderParam, observedTimestamp: number, orderType: OrderType, - source: string, idexOrder: [string, BigNumber], ): TokenOrder { const tokenOrder = new TokenOrder(); const price = new BigNumber(idexOrder[0]); const amount = idexOrder[1]; - tokenOrder.source = source; + tokenOrder.source = IDEX_SOURCE; tokenOrder.observedTimestamp = observedTimestamp; tokenOrder.orderType = orderType; tokenOrder.price = price; diff --git a/packages/pipeline/src/parsers/oasis_orders/index.ts b/packages/pipeline/src/parsers/oasis_orders/index.ts index bb3ca97fa6..25ec002d34 100644 --- a/packages/pipeline/src/parsers/oasis_orders/index.ts +++ b/packages/pipeline/src/parsers/oasis_orders/index.ts @@ -3,7 +3,7 @@ import * as R from 'ramda'; import { aggregateOrders } from '../utils'; -import { OasisMarket, OasisOrder } from '../../data_sources/oasis'; +import { OASIS_SOURCE, OasisMarket, OasisOrder } from '../../data_sources/oasis'; import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; @@ -15,21 +15,19 @@ import { OrderType } from '../../types'; * @param oasisOrderbook A raw orderbook that we pull from the Oasis API. * @param oasisMarket An object containing market data also directly from the API. * @param observedTimestamp Time at which the orders for the market were pulled. - * @param source The exchange where these orders are placed. In this case 'oasis'. */ export function parseOasisOrders( oasisOrderbook: OasisOrder[], oasisMarket: OasisMarket, observedTimestamp: number, - source: string, ): TokenOrder[] { const aggregatedBids = aggregateOrders(R.filter(R.propEq('act', OrderType.Bid), oasisOrderbook)); const aggregatedAsks = aggregateOrders(R.filter(R.propEq('act', OrderType.Ask), oasisOrderbook)); const parsedBids = aggregatedBids.map(order => - parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Bid, source, order), + parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Bid, order), ); const parsedAsks = aggregatedAsks.map(order => - parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Ask, source, order), + parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Ask, order), ); return parsedBids.concat(parsedAsks); } @@ -48,14 +46,13 @@ export function parseOasisOrder( oasisMarket: OasisMarket, observedTimestamp: number, orderType: OrderType, - source: string, oasisOrder: [string, BigNumber], ): TokenOrder { const tokenOrder = new TokenOrder(); const price = new BigNumber(oasisOrder[0]); const amount = oasisOrder[1]; - tokenOrder.source = source; + tokenOrder.source = OASIS_SOURCE; tokenOrder.observedTimestamp = observedTimestamp; tokenOrder.orderType = orderType; tokenOrder.price = price; diff --git a/packages/pipeline/src/parsers/paradex_orders/index.ts b/packages/pipeline/src/parsers/paradex_orders/index.ts index 20e442d5a9..756bb353f1 100644 --- a/packages/pipeline/src/parsers/paradex_orders/index.ts +++ b/packages/pipeline/src/parsers/paradex_orders/index.ts @@ -1,6 +1,6 @@ import { BigNumber } from '@0x/utils'; -import { ParadexMarket, ParadexOrder, ParadexOrderbookResponse } from '../../data_sources/paradex'; +import { PARADEX_SOURCE, ParadexMarket, ParadexOrder, ParadexOrderbookResponse } from '../../data_sources/paradex'; import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; @@ -12,19 +12,17 @@ import { OrderType } from '../../types'; * @param paradexOrderbookResponse An orderbook response from the Paradex API. * @param paradexMarket An object containing market data also directly from the API. * @param observedTimestamp Time at which the orders for the market were pulled. - * @param source The exchange where these orders are placed. In this case 'paradex'. */ export function parseParadexOrders( paradexOrderbookResponse: ParadexOrderbookResponse, paradexMarket: ParadexMarket, observedTimestamp: number, - source: string, ): TokenOrder[] { const parsedBids = paradexOrderbookResponse.bids.map(order => - parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Bid, source, order), + parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Bid, order), ); const parsedAsks = paradexOrderbookResponse.asks.map(order => - parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Ask, source, order), + parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Ask, order), ); return parsedBids.concat(parsedAsks); } @@ -36,21 +34,19 @@ export function parseParadexOrders( * orders have been placed. * @param observedTimestamp The time when the API response returned back to us. * @param orderType 'bid' or 'ask' enum. - * @param source Exchange where these orders were placed. * @param paradexOrder A ParadexOrder object; basically price, amount tuple. */ export function parseParadexOrder( paradexMarket: ParadexMarket, observedTimestamp: number, orderType: OrderType, - source: string, paradexOrder: ParadexOrder, ): TokenOrder { const tokenOrder = new TokenOrder(); const price = new BigNumber(paradexOrder.price); const amount = new BigNumber(paradexOrder.amount); - tokenOrder.source = source; + tokenOrder.source = PARADEX_SOURCE; tokenOrder.observedTimestamp = observedTimestamp; tokenOrder.orderType = orderType; tokenOrder.price = price; diff --git a/packages/pipeline/src/parsers/radar_orders/index.ts b/packages/pipeline/src/parsers/radar_orders/index.ts index 20472766d1..6ecedc0c36 100644 --- a/packages/pipeline/src/parsers/radar_orders/index.ts +++ b/packages/pipeline/src/parsers/radar_orders/index.ts @@ -5,6 +5,7 @@ import * as R from 'ramda'; import { aggregateOrders, GenericRawOrder } from '../utils'; +import { RADAR_SOURCE } from '../../data_sources/radar'; import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; @@ -21,21 +22,19 @@ export interface AggregateOrdersByMaker { * @param radarOrderbook A raw orderbook that we pull from the radar API. * @param radarMarket An object containing market data also directly from the API. * @param observedTimestamp Time at which the orders for the market were pulled. - * @param source The exchange where these orders are placed. In this case 'radar'. */ export function parseRadarOrders( radarOrderbook: RadarBook, radarMarket: RadarMarket, observedTimestamp: number, - source: string, ): TokenOrder[] { const aggregatedBids = _aggregateOrdersByMaker(radarMarket, radarOrderbook.bids); const aggregatedAsks = _aggregateOrdersByMaker(radarMarket, radarOrderbook.asks); const parsedBids = aggregatedBids.map(order => - parseRadarOrder(radarMarket, observedTimestamp, OrderType.Bid, source, order), + parseRadarOrder(radarMarket, observedTimestamp, OrderType.Bid, order), ); const parsedAsks = aggregatedAsks.map(order => - parseRadarOrder(radarMarket, observedTimestamp, OrderType.Ask, source, order), + parseRadarOrder(radarMarket, observedTimestamp, OrderType.Ask, order), ); return parsedBids.concat(parsedAsks); } @@ -47,14 +46,12 @@ export function parseRadarOrders( * trades have been placed. * @param observedTimestamp The time when the API response returned back to us. * @param orderType 'bid' or 'ask' enum. - * @param source Exchange where these orders were placed. * @param aggregateOrder An AggregateOrdersByMaker instance which we will convert to volume-basis. */ export function parseRadarOrder( radarMarket: RadarMarket, observedTimestamp: number, orderType: OrderType, - source: string, aggregateOrder: AggregateOrdersByMaker, ): TokenOrder { const tokenOrder = new TokenOrder(); @@ -62,7 +59,7 @@ export function parseRadarOrder( const amount = aggregateOrder.amount; const splitId = radarMarket.id.split('-'); - tokenOrder.source = source; + tokenOrder.source = RADAR_SOURCE; tokenOrder.observedTimestamp = observedTimestamp; tokenOrder.orderType = orderType; tokenOrder.price = price; diff --git a/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts index 4e00f258f3..96d82be5a2 100644 --- a/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts +++ b/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts @@ -2,7 +2,7 @@ import { logUtils } from '@0x/utils'; import * as R from 'ramda'; import { Connection, ConnectionOptions, createConnection } from 'typeorm'; -import { DDEX_SOURCE, DdexMarket, DdexSource } from '../data_sources/ddex'; +import { DdexMarket, DdexSource } from '../data_sources/ddex'; import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; import * as ormConfig from '../ormconfig'; import { parseDdexOrders } from '../parsers/ddex_orders'; @@ -43,7 +43,7 @@ async function getAndSaveMarketOrderbookAsync(ddexSource: DdexSource, market: Dd const observedTimestamp = Date.now(); logUtils.log(`${market.id}: Parsing orders.`); - const orders = parseDdexOrders(orderBook, market, observedTimestamp, DDEX_SOURCE); + const orders = parseDdexOrders(orderBook, market, observedTimestamp); if (orders.length > 0) { logUtils.log(`${market.id}: Saving ${orders.length} orders.`); diff --git a/packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts index 490b17766c..42e9fa5f09 100644 --- a/packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts +++ b/packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts @@ -2,7 +2,7 @@ import { logUtils } from '@0x/utils'; import * as R from 'ramda'; import { Connection, ConnectionOptions, createConnection } from 'typeorm'; -import { IDEX_SOURCE, IdexSource } from '../data_sources/idex'; +import { IdexSource } from '../data_sources/idex'; import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; import * as ormConfig from '../ormconfig'; import { parseIdexOrders } from '../parsers/idex_orders'; @@ -51,7 +51,7 @@ async function getAndSaveMarketOrderbookAsync(idexSource: IdexSource, marketId: } logUtils.log(`${marketId}: Parsing orders.`); - const orders = parseIdexOrders(orderBook, observedTimestamp, IDEX_SOURCE); + const orders = parseIdexOrders(orderBook, observedTimestamp); if (orders.length > 0) { logUtils.log(`${marketId}: Saving ${orders.length} orders.`); diff --git a/packages/pipeline/src/scripts/pull_oasis_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_oasis_orderbook_snapshots.ts index c4dcf6c83d..c3c286a0f9 100644 --- a/packages/pipeline/src/scripts/pull_oasis_orderbook_snapshots.ts +++ b/packages/pipeline/src/scripts/pull_oasis_orderbook_snapshots.ts @@ -2,7 +2,7 @@ import { logUtils } from '@0x/utils'; import * as R from 'ramda'; import { Connection, ConnectionOptions, createConnection } from 'typeorm'; -import { OASIS_SOURCE, OasisMarket, OasisSource } from '../data_sources/oasis'; +import { OasisMarket, OasisSource } from '../data_sources/oasis'; import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; import * as ormConfig from '../ormconfig'; import { parseOasisOrders } from '../parsers/oasis_orders'; @@ -46,7 +46,7 @@ async function getAndSaveMarketOrderbookAsync(oasisSource: OasisSource, market: const observedTimestamp = Date.now(); logUtils.log(`${market.id}: Parsing orders.`); - const orders = parseOasisOrders(orderBook, market, observedTimestamp, OASIS_SOURCE); + const orders = parseOasisOrders(orderBook, market, observedTimestamp); if (orders.length > 0) { logUtils.log(`${market.id}: Saving ${orders.length} orders.`); diff --git a/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts index 34345f3557..7fc565ca04 100644 --- a/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts +++ b/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts @@ -2,7 +2,6 @@ import { logUtils } from '@0x/utils'; import { Connection, ConnectionOptions, createConnection } from 'typeorm'; import { - PARADEX_SOURCE, ParadexActiveMarketsResponse, ParadexMarket, ParadexSource, @@ -75,7 +74,7 @@ async function getAndSaveMarketOrderbookAsync(paradexSource: ParadexSource, mark const observedTimestamp = Date.now(); logUtils.log(`${market.symbol}: Parsing orders.`); - const orders = parseParadexOrders(paradexOrderbookResponse, market, observedTimestamp, PARADEX_SOURCE); + const orders = parseParadexOrders(paradexOrderbookResponse, market, observedTimestamp); if (orders.length > 0) { logUtils.log(`${market.symbol}: Saving ${orders.length} orders.`); diff --git a/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts index 91d33d2cc9..18f5a54b20 100644 --- a/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts +++ b/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts @@ -3,7 +3,7 @@ import { RadarMarket } from '@radarrelay/types'; import * as R from 'ramda'; import { Connection, ConnectionOptions, createConnection } from 'typeorm'; -import { RADAR_SOURCE, RadarSource } from '../data_sources/radar'; +import { RadarSource } from '../data_sources/radar'; import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; import * as ormConfig from '../ormconfig'; import { parseRadarOrders } from '../parsers/radar_orders'; @@ -44,7 +44,7 @@ async function getAndSaveMarketOrderbookAsync(radarSource: RadarSource, market: const observedTimestamp = Date.now(); logUtils.log(`${market.id}: Parsing orders.`); - const orders = parseRadarOrders(orderBook, market, observedTimestamp, RADAR_SOURCE); + const orders = parseRadarOrders(orderBook, market, observedTimestamp); if (orders.length > 0) { logUtils.log(`${market.id}: Saving ${orders.length} orders.`); diff --git a/packages/pipeline/test/data_sources/radar/index_test.ts b/packages/pipeline/test/data_sources/radar/index_test.ts index 9a1eacb164..8d82ff6bf5 100644 --- a/packages/pipeline/test/data_sources/radar/index_test.ts +++ b/packages/pipeline/test/data_sources/radar/index_test.ts @@ -2,7 +2,6 @@ import { BigNumber } from '@0x/utils'; import { RadarOrderState, RadarOrderType } from '@radarrelay/types'; import * as chai from 'chai'; import 'mocha'; -import * as R from 'ramda'; import { RadarSource } from '../../../src/data_sources/radar'; import { chaiSetup } from '../../utils/chai_setup'; diff --git a/packages/pipeline/test/parsers/ddex_orders/index_test.ts b/packages/pipeline/test/parsers/ddex_orders/index_test.ts index 7eb74ed09e..de71c9c26f 100644 --- a/packages/pipeline/test/parsers/ddex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/ddex_orders/index_test.ts @@ -31,7 +31,6 @@ describe('ddex_orders', () => { }; const observedTimestamp: number = Date.now(); const orderType: OrderType = OrderType.Bid; - const source: string = 'ddex'; const expected = new TokenOrder(); expected.source = 'ddex'; @@ -45,7 +44,7 @@ describe('ddex_orders', () => { expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; expected.baseVolume = new BigNumber(10); expected.makerAddress = 'unknown'; - const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, source, ddexOrder); + const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, ddexOrder); expect(actual).deep.equal(expected); }); }); diff --git a/packages/pipeline/test/parsers/idex_orders/index_test.ts b/packages/pipeline/test/parsers/idex_orders/index_test.ts index 0c7c642ce2..7ba3adbaea 100644 --- a/packages/pipeline/test/parsers/idex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/idex_orders/index_test.ts @@ -32,7 +32,6 @@ describe('idex_orders', () => { }; const observedTimestamp: number = Date.now(); const orderType: OrderType = OrderType.Bid; - const source: string = 'idex'; const expected = new TokenOrder(); expected.source = 'idex'; @@ -46,7 +45,7 @@ describe('idex_orders', () => { expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; expected.quoteVolume = new BigNumber(5); expected.makerAddress = 'unknown'; - const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, source, idexOrder); + const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, idexOrder); expect(actual).deep.equal(expected); }); it('correctly converts ask type idexOrder to TokenOrder entity', () => { @@ -66,7 +65,6 @@ describe('idex_orders', () => { }; const observedTimestamp: number = Date.now(); const orderType: OrderType = OrderType.Ask; - const source: string = 'idex'; const expected = new TokenOrder(); expected.source = 'idex'; @@ -80,7 +78,7 @@ describe('idex_orders', () => { expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; expected.quoteVolume = new BigNumber(5); expected.makerAddress = 'unknown'; - const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, source, idexOrder); + const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, idexOrder); expect(actual).deep.equal(expected); }); }); diff --git a/packages/pipeline/test/parsers/oasis_orders/index_test.ts b/packages/pipeline/test/parsers/oasis_orders/index_test.ts index 7ef86ee10c..08ff1ef450 100644 --- a/packages/pipeline/test/parsers/oasis_orders/index_test.ts +++ b/packages/pipeline/test/parsers/oasis_orders/index_test.ts @@ -28,7 +28,6 @@ describe('oasis_orders', () => { }; const observedTimestamp: number = Date.now(); const orderType: OrderType = OrderType.Bid; - const source: string = 'oasis'; const expected = new TokenOrder(); expected.source = 'oasis'; @@ -42,7 +41,7 @@ describe('oasis_orders', () => { expected.quoteAssetAddress = null; expected.quoteVolume = new BigNumber(5); expected.makerAddress = 'unknown'; - const actual = parseOasisOrder(oasisMarket, observedTimestamp, orderType, source, oasisOrder); + const actual = parseOasisOrder(oasisMarket, observedTimestamp, orderType, oasisOrder); expect(actual).deep.equal(expected); }); }); diff --git a/packages/pipeline/test/parsers/paradex_orders/index_test.ts b/packages/pipeline/test/parsers/paradex_orders/index_test.ts index 10ac1ecdc9..5379153387 100644 --- a/packages/pipeline/test/parsers/paradex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/paradex_orders/index_test.ts @@ -33,7 +33,6 @@ describe('paradex_orders', () => { }; const observedTimestamp: number = Date.now(); const orderType: OrderType = OrderType.Bid; - const source: string = 'paradex'; const expected = new TokenOrder(); expected.source = 'paradex'; @@ -47,7 +46,7 @@ describe('paradex_orders', () => { expected.quoteAssetAddress = '0x0000000000000000000000000000000000000000'; expected.quoteVolume = new BigNumber(412 * 0.1245); expected.makerAddress = 'unknown'; - const actual = parseParadexOrder(paradexMarket, observedTimestamp, orderType, source, paradexOrder); + const actual = parseParadexOrder(paradexMarket, observedTimestamp, orderType, paradexOrder); expect(actual).deep.equal(expected); }); }); diff --git a/packages/pipeline/test/parsers/radar_orders/index_test.ts b/packages/pipeline/test/parsers/radar_orders/index_test.ts index e3004d7c2a..29ae41a00d 100644 --- a/packages/pipeline/test/parsers/radar_orders/index_test.ts +++ b/packages/pipeline/test/parsers/radar_orders/index_test.ts @@ -1,5 +1,5 @@ import { BigNumber } from '@0x/utils'; -import { RadarBook, RadarMarket, RadarSignedOrder } from '@radarrelay/types'; +import { RadarMarket } from '@radarrelay/types'; import * as chai from 'chai'; import 'mocha'; @@ -35,7 +35,6 @@ describe('radar_orders', () => { } as any) as RadarMarket; const observedTimestamp: number = Date.now(); const orderType: OrderType = OrderType.Bid; - const source: string = 'radar'; const expected = new TokenOrder(); expected.source = 'radar'; @@ -49,7 +48,7 @@ describe('radar_orders', () => { expected.baseAssetAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; expected.baseVolume = new BigNumber(10000000000); expected.makerAddress = '0x6eC92694ea172ebC430C30fa31De87620967A082'; - const actual = parseRadarOrder(radarMarket, observedTimestamp, orderType, source, radarOrder); + const actual = parseRadarOrder(radarMarket, observedTimestamp, orderType, radarOrder); expect(actual).deep.equal(expected); }); });