diff --git a/packages/pipeline/migrations/1543540108767-CreateNftTrades.ts b/packages/pipeline/migrations/1543540108767-CreateNftTrades.ts new file mode 100644 index 0000000000..a35e8aee24 --- /dev/null +++ b/packages/pipeline/migrations/1543540108767-CreateNftTrades.ts @@ -0,0 +1,31 @@ +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; + +const nftTrades = new Table({ + name: 'raw.nonfungible_dot_com_trades', + columns: [ + { name: 'publisher', type: 'varchar', isPrimary: true }, + { name: 'transaction_hash', type: 'varchar', isPrimary: true }, + { name: 'asset_id', type: 'varchar', isPrimary: true }, + { name: 'block_number', type: 'bigint', isPrimary: true }, + { name: 'log_index', type: 'integer', isPrimary: true }, + + { name: 'block_timestamp', type: 'bigint' }, + { name: 'asset_descriptor', type: 'varchar' }, + { name: 'market_address', type: 'varchar(42)' }, + { name: 'total_price', type: 'numeric' }, + { name: 'usd_price', type: 'numeric' }, + { name: 'buyer_address', type: 'varchar(42)' }, + { name: 'seller_address', type: 'varchar(42)' }, + { name: 'meta', type: 'jsonb' }, + ], +}); + +export class CreateNftTrades1543540108767 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable(nftTrades); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable(nftTrades); + } +} diff --git a/packages/pipeline/migrations/1549856835629-CreateSlippageTable.ts b/packages/pipeline/migrations/1549856835629-CreateSlippageTable.ts new file mode 100644 index 0000000000..7dfe9fc7fe --- /dev/null +++ b/packages/pipeline/migrations/1549856835629-CreateSlippageTable.ts @@ -0,0 +1,26 @@ +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; + +const slippage = new Table({ + name: 'raw.slippage', + columns: [ + { name: 'observed_timestamp', type: 'bigint', isPrimary: true }, + { name: 'symbol', type: 'varchar', isPrimary: true }, + { name: 'exchange', type: 'varchar', isPrimary: true }, + { name: 'usd_amount', type: 'numeric', isPrimary: true }, + + { name: 'token_amount', type: 'numeric', isNullable: false }, + { name: 'avg_price_in_eth_buy', type: 'numeric', isNullable: true }, + { name: 'avg_price_in_eth_sell', type: 'numeric', isNullable: true }, + { name: 'slippage', type: 'numeric', isNullable: true }, + ], +}); + +export class CreateSlippageTable1549856835629 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable(slippage); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable(slippage); + } +} diff --git a/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts b/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts new file mode 100644 index 0000000000..1d3b3c3ab5 --- /dev/null +++ b/packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts @@ -0,0 +1,40 @@ +import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm'; + +const TOKEN_ORDERBOOK_SNAPSHOT_TABLE = 'raw.token_orderbook_snapshots'; +const NEW_COLUMN_NAME = 'maker_address'; + +export class TokenOrderBookSnapshotsAddMakerAddress1550163069315 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + const snapshotTable = await queryRunner.getTable(TOKEN_ORDERBOOK_SNAPSHOT_TABLE); + if (snapshotTable) { + const newColumn = new TableColumn({ + name: NEW_COLUMN_NAME, + type: 'varchar', + isNullable: true, + }); + await queryRunner.addColumn(TOKEN_ORDERBOOK_SNAPSHOT_TABLE, newColumn); + // backfill null values + await queryRunner.query(` + UPDATE ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE} + SET ${NEW_COLUMN_NAME}='unknown' + WHERE ${NEW_COLUMN_NAME} is NULL; + `); + await queryRunner.query(` + ALTER TABLE ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE} + DROP CONSTRAINT "token_orderbook_snapshots_pkey1", + ADD PRIMARY KEY (observed_timestamp, source, order_type, price, base_asset_symbol, quote_asset_symbol, maker_address); + `); + } else { + throw new Error(`Could not find table with name ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE}`); + } + } + + public async down(queryRunner: QueryRunner): Promise { + const snapshotTable = await queryRunner.getTable(TOKEN_ORDERBOOK_SNAPSHOT_TABLE); + if (snapshotTable) { + await queryRunner.dropColumn(snapshotTable, NEW_COLUMN_NAME); + } else { + throw new Error(`Could not find table with name ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE}`); + } + } +} diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 56d5ef10d9..21bea768e6 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -49,6 +49,7 @@ "@0x/types": "^2.0.2", "@0x/utils": "^4.1.0", "@0x/web3-wrapper": "^5.0.0", + "@radarrelay/types": "^1.2.1", "@types/dockerode": "^2.5.9", "@types/p-limit": "^2.0.0", "async-parallel": "^1.2.3", diff --git a/packages/pipeline/src/data_sources/dex_prices/index.ts b/packages/pipeline/src/data_sources/dex_prices/index.ts new file mode 100644 index 0000000000..3796aaa75b --- /dev/null +++ b/packages/pipeline/src/data_sources/dex_prices/index.ts @@ -0,0 +1,41 @@ +import { fetchAsync } from '@0x/utils'; + +const EDPS_BASE_URL = 'http://35.185.219.196:1337'; + +export type EdpsResponse = EdpsWrapper[]; + +export interface EdpsWrapper { + [key: string]: EdpsExchange; +} + +export interface EdpsExchange { + exchangeName: string; + totalPrice: number; + tokenAmount: number; + tokenSymbol: string; + avgPrice: number; + timestamp: number; + error: string; +} + +// tslint:disable:prefer-function-over-method +// ^ Keep consistency with other sources and help logical organization +export class EdpsSource { + /** + * Call Ethereum DEX Price Service API. + */ + public async getEdpsAsync(direction: string, symbol: string, amount: number): Promise { + const edpsUrl = `${EDPS_BASE_URL}/${direction}?amount=${amount}&symbol=${symbol}&decimals=`; + const resp = await fetchAsync(edpsUrl); + const respJson: EdpsResponse = await resp.json(); + const allExchanges: EdpsWrapper = {}; + // The below unwraps the response so we get 1 single EdpsWrapper object + // instead of a list of singletons + for (const entry of respJson) { + for (const key of Object.keys(entry)) { + allExchanges[key] = entry[key]; + } + } + return allExchanges; + } +} diff --git a/packages/pipeline/src/data_sources/nonfungible_dot_com/index.ts b/packages/pipeline/src/data_sources/nonfungible_dot_com/index.ts new file mode 100644 index 0000000000..7ec701ba62 --- /dev/null +++ b/packages/pipeline/src/data_sources/nonfungible_dot_com/index.ts @@ -0,0 +1,220 @@ +import { stringify } from 'querystring'; + +import { logUtils } from '@0x/utils'; + +import { fetchSuccessfullyOrThrowAsync } from '../../utils'; + +// URL to use for getting nft trades from nonfungible.com. +export const NONFUNGIBLE_DOT_COM_URL = 'https://nonfungible.com/api/v1'; +// Number of trades to get at once. This is a hard limit enforced by the API. +const MAX_TRADES_PER_QUERY = 100; + +// Note(albrow): For now this will have to be manually updated by checking +// https://nonfungible.com/ +export const knownPublishers = [ + 'axieinfinity', + // 'cryptokitties', // disabled until we get updated initial dump that isn't truncated + 'cryptopunks', + 'cryptovoxels', + 'decentraland', + 'decentraland_estate', + 'etherbots', + 'etheremon', + 'ethtown', + // 'knownorigin', // disabled because of null characters in data being rejected by postgres + // 'mythereum', // worked at one time, but now seems dead + 'superrare', +]; + +export interface NonfungibleDotComHistoryResponse { + data: NonfungibleDotComTradeResponse[]; +} + +export interface NonfungibleDotComTradeResponse { + _id: string; + transactionHash: string; + blockNumber: number; + logIndex: number; + blockTimestamp: string; + assetId: string; + assetDescriptor: string; + nftAddress: string; + marketAddress: string; + tokenTicker: string; + totalDecimalPrice: number; + totalPrice: string; + usdPrice: number; + currencyTransfer: object; + buyer: string; + seller: string; + meta: object; + image: string; + composedOf: string; + asset_link: string; + seller_address_link: string; + buyer_address_link: string; +} + +/** + * Gets and returns all trades for the given publisher, starting at the given block number. + * Automatically handles pagination. + * @param publisher A valid "publisher" for the nonfungible.com API. (e.g. "cryptokitties") + * @param blockNumberStart The block number to start querying from. + */ +export async function getTradesAsync( + publisher: string, + blockNumberStart: number, +): Promise { + const allTrades: NonfungibleDotComTradeResponse[] = []; + + /** + * due to high data volumes and rate limiting, we procured an initial data + * dump from nonfungible.com. If the requested starting block number is + * contained in that initial dump, then pull relevant trades from there + * first. Later (below) we'll get the more recent trades from the API itself. + */ + + if (blockNumberStart < highestBlockNumbersInIntialDump[publisher]) { + logUtils.log('getting trades from one-time dump'); + // caller needs trades that are in the initial data dump, so get them + // from there, then later go to the API for the rest. + const initialDumpResponse: NonfungibleDotComHistoryResponse = await fetchSuccessfullyOrThrowAsync( + getInitialDumpUrl(publisher), + ); + const initialDumpTrades = initialDumpResponse.data; + for (const initialDumpTrade of initialDumpTrades) { + if (!shouldProcessTrade(initialDumpTrade, allTrades)) { + continue; + } + + ensureNonNull(initialDumpTrade); + + allTrades.push(initialDumpTrade); + } + logUtils.log(`got ${allTrades.length} from one-time dump`); + } + + const fullUrl = getFullUrlForPublisher(publisher); + + /** + * API returns trades in reverse chronological order, so highest block + * numbers first. The `start` query parameter indicates how far back in + * time (in number of trades) the results should start. Here we iterate + * over both start parameter values and block numbers simultaneously. + * Start parameter values count up from zero. Block numbers count down + * until reaching the highest block number in the initial dump. + */ + + const blockNumberStop = Math.max(highestBlockNumbersInIntialDump[publisher] + 1, blockNumberStart); + for ( + let startParam = 0, blockNumber = Number.MAX_SAFE_INTEGER; + blockNumber > blockNumberStop; + startParam += MAX_TRADES_PER_QUERY + ) { + const response = await _getTradesWithOffsetAsync(fullUrl, publisher, startParam); + const tradesFromApi = response.data; + logUtils.log( + `got ${ + tradesFromApi.length + } trades from API. blockNumber=${blockNumber}. blockNumberStop=${blockNumberStop}`, + ); + for (const tradeFromApi of tradesFromApi) { + if (tradeFromApi.blockNumber <= blockNumberStop) { + blockNumber = blockNumberStop; + break; + } + if (!shouldProcessTrade(tradeFromApi, allTrades)) { + continue; + } + ensureNonNull(tradeFromApi); + allTrades.push(tradeFromApi); + blockNumber = tradeFromApi.blockNumber; + } + } + + return allTrades; +} + +function shouldProcessTrade( + trade: NonfungibleDotComTradeResponse, + existingTrades: NonfungibleDotComTradeResponse[], +): boolean { + // check to see if this trade is already in existingTrades + const existingTradeIndex = existingTrades.findIndex( + // HACK! making assumptions about composition of primary key + e => + e.transactionHash === trade.transactionHash && + e.logIndex === trade.logIndex && + e.blockNumber === trade.blockNumber, + ); + if (existingTradeIndex !== -1) { + logUtils.log("we've already captured this trade. deciding whether to use the existing record or this one."); + if (trade.blockNumber > existingTrades[existingTradeIndex].blockNumber) { + logUtils.log('throwing out existing trade'); + existingTrades.splice(existingTradeIndex, 1); + } else { + logUtils.log('letting existing trade stand, and skipping processing of this trade'); + return false; + } + } + return true; +} + +const highestBlockNumbersInIntialDump: { [publisher: string]: number } = { + axieinfinity: 7065913, + cryptokitties: 4658171, + cryptopunks: 7058897, + cryptovoxels: 7060783, + decentraland_estate: 7065181, + decentraland: 6938962, + etherbots: 5204980, + etheremon: 7065370, + ethtown: 7064126, + knownorigin: 7065160, + mythereum: 7065311, + superrare: 7065955, +}; + +async function _getTradesWithOffsetAsync( + url: string, + publisher: string, + offset: number, +): Promise { + const resp: NonfungibleDotComHistoryResponse = await fetchSuccessfullyOrThrowAsync( + `${url}?${stringify({ + publisher, + start: offset, + length: MAX_TRADES_PER_QUERY, + })}`, + ); + return resp; +} + +function getFullUrlForPublisher(publisher: string): string { + return `${NONFUNGIBLE_DOT_COM_URL}/market/${publisher}/history`; +} + +function getInitialDumpUrl(publisher: string): string { + return `https://nonfungible-dot-com-one-time-data-dump.s3.amazonaws.com/sales_summary_${publisher}.json`; +} + +function ensureNonNull(trade: NonfungibleDotComTradeResponse): void { + // these fields need to be set in order to avoid non-null + // constraint exceptions upon database insertion. + if (trade.logIndex === undefined) { + // for cryptopunks + trade.logIndex = 0; + } + if (trade.assetDescriptor === undefined) { + // for cryptopunks + trade.assetDescriptor = ''; + } + if (trade.meta === undefined) { + // for cryptopunks + trade.meta = {}; + } + if (trade.marketAddress === null) { + // for decentraland_estate + trade.marketAddress = ''; + } +} diff --git a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts index 85042501b1..961e6ff61a 100644 --- a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts +++ b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts @@ -33,6 +33,10 @@ export interface CryptoCompareOHLCVParams { toTs?: number; } +export interface CryptoCompareUsdPrice { + USD: number; +} + const ONE_HOUR = 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers const ONE_SECOND = 1000; const ONE_HOUR_AGO = new Date().getTime() - ONE_HOUR; @@ -45,6 +49,7 @@ export class CryptoCompareOHLCVSource { public readonly defaultExchange = 'CCCAGG'; public readonly interval = this.intervalBetweenRecords * MAX_PAGE_SIZE; // the hourly API returns data for one interval at a time private readonly _url: string = 'https://min-api.cryptocompare.com/data/histohour?'; + private readonly _priceUrl: string = 'https://min-api.cryptocompare.com/data/price?'; // rate-limit for all API calls through this class instance private readonly _limiter: Bottleneck; @@ -96,6 +101,13 @@ export class CryptoCompareOHLCVSource { }; return R.unfold(f, pair); } + + public async getUsdPriceAsync(symbol: string): Promise { + const usdUrl = `${this._priceUrl}tsyms=USD&fsym=${symbol}`; + const resp = await fetchAsync(usdUrl); + const respJson: CryptoCompareUsdPrice = await resp.json(); + return respJson.USD; + } } function hasData(record: CryptoCompareOHLCVRecord): boolean { diff --git a/packages/pipeline/src/data_sources/radar/index.ts b/packages/pipeline/src/data_sources/radar/index.ts new file mode 100644 index 0000000000..873b0fefe2 --- /dev/null +++ b/packages/pipeline/src/data_sources/radar/index.ts @@ -0,0 +1,53 @@ +import { orderParsingUtils } from '@0x/order-utils'; +import { fetchAsync, logUtils } from '@0x/utils'; +import { RadarBook, RadarMarket, RadarSignedOrder } from '@radarrelay/types'; + +const RADAR_BASE_URL = 'https://api.radarrelay.com/v2/'; +const ACTIVE_MARKETS_URL = `${RADAR_BASE_URL}/markets`; +const MAX_PER_PAGE = 10000; + +export const RADAR_SOURCE = 'radar'; + +// tslint:disable:prefer-function-over-method +// ^ Keep consistency with other sources and help logical organization +export class RadarSource { + public static parseRadarOrderResponse(radarOrderResponse: any): RadarSignedOrder { + return { + ...radarOrderResponse, + ...orderParsingUtils.convertStringsFieldsToBigNumbers(radarOrderResponse, [ + 'remainingBaseTokenAmount', + 'remainingQuoteTokenAmount', + 'price', + ]), + signedOrder: orderParsingUtils.convertOrderStringFieldsToBigNumber(radarOrderResponse.signedOrder), + }; + } + /** + * Call Radar API to find out which markets they are maintaining orderbooks for. + */ + public async getActiveMarketsAsync(): Promise { + logUtils.log('Getting all active Radar markets'); + const resp = await fetchAsync(`${ACTIVE_MARKETS_URL}?perPage=${MAX_PER_PAGE}`); + const markets: RadarMarket[] = await resp.json(); + logUtils.log(`Got ${markets.length} markets.`); + return markets; + } + + /** + * Retrieve orderbook from Radar API for a given market. + * @param marketId String identifying the market we want data for. Eg. 'REP/AUG' + */ + public async getMarketOrderbookAsync(marketId: string): Promise { + logUtils.log(`${marketId}: Retrieving orderbook.`); + const marketOrderbookUrl = `${ACTIVE_MARKETS_URL}/${marketId}/book?perPage=${MAX_PER_PAGE}`; + const resp = await fetchAsync(marketOrderbookUrl); + const jsonResp = await resp.json(); + return { + ...jsonResp, + // tslint:disable-next-line:no-unbound-method + bids: jsonResp.bids.map(RadarSource.parseRadarOrderResponse), + // tslint:disable-next-line:no-unbound-method + asks: jsonResp.asks.map(RadarSource.parseRadarOrderResponse), + }; + } +} diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts index 27c153c079..174a32a446 100644 --- a/packages/pipeline/src/entities/index.ts +++ b/packages/pipeline/src/entities/index.ts @@ -7,8 +7,10 @@ export { DexTrade } from './dex_trade'; export { ExchangeCancelEvent } from './exchange_cancel_event'; export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event'; export { ExchangeFillEvent } from './exchange_fill_event'; +export { NonfungibleDotComTrade } from './nonfungible_dot_com_trade'; export { OHLCVExternal } from './ohlcv_external'; export { Relayer } from './relayer'; +export { Slippage } from './slippage'; export { SraOrder } from './sra_order'; export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp'; export { TokenMetadata } from './token_metadata'; diff --git a/packages/pipeline/src/entities/nonfungible_dot_com_trade.ts b/packages/pipeline/src/entities/nonfungible_dot_com_trade.ts new file mode 100644 index 0000000000..514edafcb2 --- /dev/null +++ b/packages/pipeline/src/entities/nonfungible_dot_com_trade.ts @@ -0,0 +1,35 @@ +import { BigNumber } from '@0x/utils'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'nonfungible_dot_com_trades', schema: 'raw' }) +export class NonfungibleDotComTrade { + @PrimaryColumn({ name: 'transaction_hash' }) + public transactionHash!: string; + @PrimaryColumn({ name: 'publisher' }) + public publisher!: string; + @PrimaryColumn({ name: 'block_number', type: 'bigint', transformer: numberToBigIntTransformer }) + public blockNumber!: number; + @PrimaryColumn({ name: 'log_index' }) + public logIndex!: number; + @PrimaryColumn({ name: 'asset_id' }) + public assetId!: string; + + @Column({ name: 'block_timestamp', type: 'bigint', transformer: numberToBigIntTransformer }) + public blockTimestamp!: number; + @Column({ name: 'asset_descriptor' }) + public assetDescriptor!: string; + @Column({ name: 'market_address' }) + public marketAddress!: string; + @Column({ name: 'total_price', type: 'numeric', transformer: bigNumberTransformer }) + public totalPrice!: BigNumber; + @Column({ name: 'usd_price', type: 'numeric', transformer: bigNumberTransformer }) + public usdPrice!: BigNumber; + @Column({ name: 'buyer_address' }) + public buyerAddress!: string; + @Column({ name: 'seller_address' }) + public sellerAddress!: string; + @Column({ type: 'jsonb' }) + public meta!: object; +} diff --git a/packages/pipeline/src/entities/slippage.ts b/packages/pipeline/src/entities/slippage.ts new file mode 100644 index 0000000000..87744ca048 --- /dev/null +++ b/packages/pipeline/src/entities/slippage.ts @@ -0,0 +1,25 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { BigNumber } from '@0x/utils'; + +import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'slippage', schema: 'raw' }) +export class Slippage { + @PrimaryColumn({ name: 'observed_timestamp', type: 'bigint', transformer: numberToBigIntTransformer }) + public observedTimestamp!: number; + @PrimaryColumn({ name: 'symbol' }) + public symbol!: string; + @PrimaryColumn({ name: 'exchange' }) + public exchange!: string; + @PrimaryColumn({ name: 'usd_amount', type: 'numeric', transformer: bigNumberTransformer }) + public usdAmount!: BigNumber; + @Column({ name: 'token_amount', type: 'numeric', transformer: bigNumberTransformer }) + public tokenAmount!: BigNumber; + @Column({ name: 'avg_price_in_eth_sell', type: 'numeric', transformer: bigNumberTransformer }) + public avgPriceInEthSell?: BigNumber; + @Column({ name: 'avg_price_in_eth_buy', type: 'numeric', transformer: bigNumberTransformer }) + public avgPriceInEthBuy?: BigNumber; + @Column({ name: 'slippage', type: 'numeric', transformer: bigNumberTransformer }) + public slippage?: BigNumber; +} diff --git a/packages/pipeline/src/entities/token_order.ts b/packages/pipeline/src/entities/token_order.ts index 2709747cb3..a1f0006d6e 100644 --- a/packages/pipeline/src/entities/token_order.ts +++ b/packages/pipeline/src/entities/token_order.ts @@ -15,12 +15,14 @@ export class TokenOrderbookSnapshot { public price!: BigNumber; @PrimaryColumn({ name: 'base_asset_symbol' }) public baseAssetSymbol!: string; + @PrimaryColumn({ name: 'quote_asset_symbol' }) + public quoteAssetSymbol!: string; + @PrimaryColumn({ type: String, name: 'maker_address', default: 'unknown' }) + public makerAddress!: string; @Column({ nullable: true, type: String, name: 'base_asset_address' }) public baseAssetAddress!: string | null; @Column({ name: 'base_volume', type: 'numeric', transformer: bigNumberTransformer }) public baseVolume!: BigNumber; - @PrimaryColumn({ name: 'quote_asset_symbol' }) - public quoteAssetSymbol!: string; @Column({ nullable: true, type: String, name: 'quote_asset_address' }) public quoteAssetAddress!: string | null; @Column({ name: 'quote_volume', type: 'numeric', transformer: bigNumberTransformer }) diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 2700714cdd..4604686acf 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -12,8 +12,10 @@ import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, + NonfungibleDotComTrade, OHLCVExternal, Relayer, + Slippage, SraOrder, SraOrdersObservedTimeStamp, TokenMetadata, @@ -33,8 +35,10 @@ const entities = [ ExchangeCancelUpToEvent, ExchangeFillEvent, ERC20ApprovalEvent, + NonfungibleDotComTrade, OHLCVExternal, Relayer, + Slippage, SraOrder, SraOrdersObservedTimeStamp, TokenMetadata, diff --git a/packages/pipeline/src/parsers/ddex_orders/index.ts b/packages/pipeline/src/parsers/ddex_orders/index.ts index 562f894ab6..6c98c32254 100644 --- a/packages/pipeline/src/parsers/ddex_orders/index.ts +++ b/packages/pipeline/src/parsers/ddex_orders/index.ts @@ -2,33 +2,27 @@ import { BigNumber } from '@0x/utils'; import { aggregateOrders } from '../utils'; -import { DdexMarket, DdexOrderbook } from '../../data_sources/ddex'; +import { DDEX_SOURCE, DdexMarket, DdexOrderbook } from '../../data_sources/ddex'; import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; /** - * Marque function of this file. + * Marquee function of this file. * 1) Takes in orders from an orderbook, * other information attached. * @param ddexOrderbook A raw orderbook that we pull from the Ddex API. * @param ddexMarket An object containing market data also directly from the API. * @param observedTimestamp Time at which the orders for the market were pulled. - * @param source The exchange where these orders are placed. In this case 'ddex'. */ export function parseDdexOrders( ddexOrderbook: DdexOrderbook, ddexMarket: DdexMarket, observedTimestamp: number, - source: string, ): TokenOrder[] { const aggregatedBids = aggregateOrders(ddexOrderbook.bids); const aggregatedAsks = aggregateOrders(ddexOrderbook.asks); - const parsedBids = aggregatedBids.map(order => - parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Bid, source, order), - ); - const parsedAsks = aggregatedAsks.map(order => - parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Ask, source, order), - ); + const parsedBids = aggregatedBids.map(order => parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Bid, order)); + const parsedAsks = aggregatedAsks.map(order => parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Ask, order)); return parsedBids.concat(parsedAsks); } @@ -46,14 +40,13 @@ export function parseDdexOrder( ddexMarket: DdexMarket, observedTimestamp: number, orderType: OrderType, - source: string, ddexOrder: [string, BigNumber], ): TokenOrder { const tokenOrder = new TokenOrder(); const price = new BigNumber(ddexOrder[0]); const amount = ddexOrder[1]; - tokenOrder.source = source; + tokenOrder.source = DDEX_SOURCE; tokenOrder.observedTimestamp = observedTimestamp; tokenOrder.orderType = orderType; tokenOrder.price = price; @@ -65,5 +58,7 @@ export function parseDdexOrder( tokenOrder.quoteAssetSymbol = ddexMarket.quoteToken; tokenOrder.quoteAssetAddress = ddexMarket.quoteTokenAddress; tokenOrder.quoteVolume = price.times(amount); + + tokenOrder.makerAddress = 'unknown'; return tokenOrder; } diff --git a/packages/pipeline/src/parsers/idex_orders/index.ts b/packages/pipeline/src/parsers/idex_orders/index.ts index 14b8711958..b151fda59d 100644 --- a/packages/pipeline/src/parsers/idex_orders/index.ts +++ b/packages/pipeline/src/parsers/idex_orders/index.ts @@ -2,28 +2,25 @@ import { BigNumber } from '@0x/utils'; import { aggregateOrders } from '../utils'; -import { IdexOrderbook, IdexOrderParam } from '../../data_sources/idex'; +import { IDEX_SOURCE, IdexOrderbook, IdexOrderParam } from '../../data_sources/idex'; import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; /** - * Marque function of this file. + * Marquee function of this file. * 1) Takes in orders from an orderbook, * 2) Aggregates them by price point, * 3) Parses them into entities which are then saved into the database. * @param idexOrderbook raw orderbook that we pull from the Idex API. * @param observedTimestamp Time at which the orders for the market were pulled. - * @param source The exchange where these orders are placed. In this case 'idex'. */ -export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: number, source: string): TokenOrder[] { +export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: number): TokenOrder[] { const aggregatedBids = aggregateOrders(idexOrderbook.bids); // Any of the bid orders' params will work const idexBidOrder = idexOrderbook.bids[0]; const parsedBids = aggregatedBids.length > 0 - ? aggregatedBids.map(order => - parseIdexOrder(idexBidOrder.params, observedTimestamp, OrderType.Bid, source, order), - ) + ? aggregatedBids.map(order => parseIdexOrder(idexBidOrder.params, observedTimestamp, OrderType.Bid, order)) : []; const aggregatedAsks = aggregateOrders(idexOrderbook.asks); @@ -31,9 +28,7 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: const idexAskOrder = idexOrderbook.asks[0]; const parsedAsks = aggregatedAsks.length > 0 - ? aggregatedAsks.map(order => - parseIdexOrder(idexAskOrder.params, observedTimestamp, OrderType.Ask, source, order), - ) + ? aggregatedAsks.map(order => parseIdexOrder(idexAskOrder.params, observedTimestamp, OrderType.Ask, order)) : []; return parsedBids.concat(parsedAsks); } @@ -45,26 +40,25 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: * trades have been placed. * @param observedTimestamp The time when the API response returned back to us. * @param orderType 'bid' or 'ask' enum. - * @param source Exchange where these orders were placed. * @param idexOrder A tuple which we will convert to volume-basis. */ export function parseIdexOrder( idexOrderParam: IdexOrderParam, observedTimestamp: number, orderType: OrderType, - source: string, idexOrder: [string, BigNumber], ): TokenOrder { const tokenOrder = new TokenOrder(); const price = new BigNumber(idexOrder[0]); const amount = idexOrder[1]; - tokenOrder.source = source; + tokenOrder.source = IDEX_SOURCE; tokenOrder.observedTimestamp = observedTimestamp; tokenOrder.orderType = orderType; tokenOrder.price = price; tokenOrder.baseVolume = amount; tokenOrder.quoteVolume = price.times(amount); + tokenOrder.makerAddress = 'unknown'; if (orderType === OrderType.Bid) { tokenOrder.baseAssetSymbol = idexOrderParam.buySymbol; diff --git a/packages/pipeline/src/parsers/nonfungible_dot_com/index.ts b/packages/pipeline/src/parsers/nonfungible_dot_com/index.ts new file mode 100644 index 0000000000..48daa3d7fd --- /dev/null +++ b/packages/pipeline/src/parsers/nonfungible_dot_com/index.ts @@ -0,0 +1,42 @@ +import { BigNumber } from '@0x/utils'; +import * as R from 'ramda'; + +import { NonfungibleDotComTradeResponse } from '../../data_sources/nonfungible_dot_com'; +import { NonfungibleDotComTrade } from '../../entities'; + +/** + * Parses a raw trades from the nonfungible.com API and returns an array of + * NonfungibleDotComTrade entities. + * @param rawTrades A raw order response from an SRA endpoint. + */ +export function parseNonFungibleDotComTrades( + rawTrades: NonfungibleDotComTradeResponse[], + publisher: string, +): NonfungibleDotComTrade[] { + return R.map(_parseNonFungibleDotComTrade.bind(null, publisher), rawTrades); +} + +/** + * Converts a single trade from nonfungible.com into an NonfungibleDotComTrade entity. + * @param rawTrade A single trade from the response from the nonfungible.com API. + */ +export function _parseNonFungibleDotComTrade( + publisher: string, + rawTrade: NonfungibleDotComTradeResponse, +): NonfungibleDotComTrade { + const nonfungibleDotComTrade = new NonfungibleDotComTrade(); + nonfungibleDotComTrade.assetDescriptor = rawTrade.assetDescriptor; + nonfungibleDotComTrade.assetId = rawTrade.assetId; + nonfungibleDotComTrade.blockNumber = rawTrade.blockNumber; + nonfungibleDotComTrade.blockTimestamp = new Date(rawTrade.blockTimestamp).getTime(); + nonfungibleDotComTrade.buyerAddress = rawTrade.buyer; + nonfungibleDotComTrade.logIndex = rawTrade.logIndex; + nonfungibleDotComTrade.marketAddress = rawTrade.marketAddress; + nonfungibleDotComTrade.meta = rawTrade.meta; + nonfungibleDotComTrade.sellerAddress = rawTrade.seller; + nonfungibleDotComTrade.totalPrice = new BigNumber(rawTrade.totalPrice); + nonfungibleDotComTrade.transactionHash = rawTrade.transactionHash; + nonfungibleDotComTrade.usdPrice = new BigNumber(rawTrade.usdPrice); + nonfungibleDotComTrade.publisher = publisher; + return nonfungibleDotComTrade; +} diff --git a/packages/pipeline/src/parsers/oasis_orders/index.ts b/packages/pipeline/src/parsers/oasis_orders/index.ts index b71fb65b90..25ec002d34 100644 --- a/packages/pipeline/src/parsers/oasis_orders/index.ts +++ b/packages/pipeline/src/parsers/oasis_orders/index.ts @@ -3,33 +3,31 @@ import * as R from 'ramda'; import { aggregateOrders } from '../utils'; -import { OasisMarket, OasisOrder } from '../../data_sources/oasis'; +import { OASIS_SOURCE, OasisMarket, OasisOrder } from '../../data_sources/oasis'; import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; /** - * Marque function of this file. + * Marquee function of this file. * 1) Takes in orders from an orderbook, * 2) Aggregates them according to price point, * 3) Builds TokenOrder entity with other information attached. * @param oasisOrderbook A raw orderbook that we pull from the Oasis API. * @param oasisMarket An object containing market data also directly from the API. * @param observedTimestamp Time at which the orders for the market were pulled. - * @param source The exchange where these orders are placed. In this case 'oasis'. */ export function parseOasisOrders( oasisOrderbook: OasisOrder[], oasisMarket: OasisMarket, observedTimestamp: number, - source: string, ): TokenOrder[] { const aggregatedBids = aggregateOrders(R.filter(R.propEq('act', OrderType.Bid), oasisOrderbook)); const aggregatedAsks = aggregateOrders(R.filter(R.propEq('act', OrderType.Ask), oasisOrderbook)); const parsedBids = aggregatedBids.map(order => - parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Bid, source, order), + parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Bid, order), ); const parsedAsks = aggregatedAsks.map(order => - parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Ask, source, order), + parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Ask, order), ); return parsedBids.concat(parsedAsks); } @@ -48,14 +46,13 @@ export function parseOasisOrder( oasisMarket: OasisMarket, observedTimestamp: number, orderType: OrderType, - source: string, oasisOrder: [string, BigNumber], ): TokenOrder { const tokenOrder = new TokenOrder(); const price = new BigNumber(oasisOrder[0]); const amount = oasisOrder[1]; - tokenOrder.source = source; + tokenOrder.source = OASIS_SOURCE; tokenOrder.observedTimestamp = observedTimestamp; tokenOrder.orderType = orderType; tokenOrder.price = price; @@ -67,5 +64,6 @@ export function parseOasisOrder( tokenOrder.quoteAssetSymbol = oasisMarket.quote; tokenOrder.quoteAssetAddress = null; // Oasis doesn't provide address information tokenOrder.quoteVolume = price.times(amount); + tokenOrder.makerAddress = 'unknown'; return tokenOrder; } diff --git a/packages/pipeline/src/parsers/paradex_orders/index.ts b/packages/pipeline/src/parsers/paradex_orders/index.ts index 85990dae4f..756bb353f1 100644 --- a/packages/pipeline/src/parsers/paradex_orders/index.ts +++ b/packages/pipeline/src/parsers/paradex_orders/index.ts @@ -1,30 +1,28 @@ import { BigNumber } from '@0x/utils'; -import { ParadexMarket, ParadexOrder, ParadexOrderbookResponse } from '../../data_sources/paradex'; +import { PARADEX_SOURCE, ParadexMarket, ParadexOrder, ParadexOrderbookResponse } from '../../data_sources/paradex'; import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; /** - * Marque function of this file. + * Marquee function of this file. * 1) Takes in orders from an orderbook (orders are already aggregated by price point), * 2) For each aggregated order, forms a TokenOrder entity with market data and * other information attached. * @param paradexOrderbookResponse An orderbook response from the Paradex API. * @param paradexMarket An object containing market data also directly from the API. * @param observedTimestamp Time at which the orders for the market were pulled. - * @param source The exchange where these orders are placed. In this case 'paradex'. */ export function parseParadexOrders( paradexOrderbookResponse: ParadexOrderbookResponse, paradexMarket: ParadexMarket, observedTimestamp: number, - source: string, ): TokenOrder[] { const parsedBids = paradexOrderbookResponse.bids.map(order => - parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Bid, source, order), + parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Bid, order), ); const parsedAsks = paradexOrderbookResponse.asks.map(order => - parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Ask, source, order), + parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Ask, order), ); return parsedBids.concat(parsedAsks); } @@ -36,21 +34,19 @@ export function parseParadexOrders( * orders have been placed. * @param observedTimestamp The time when the API response returned back to us. * @param orderType 'bid' or 'ask' enum. - * @param source Exchange where these orders were placed. * @param paradexOrder A ParadexOrder object; basically price, amount tuple. */ export function parseParadexOrder( paradexMarket: ParadexMarket, observedTimestamp: number, orderType: OrderType, - source: string, paradexOrder: ParadexOrder, ): TokenOrder { const tokenOrder = new TokenOrder(); const price = new BigNumber(paradexOrder.price); const amount = new BigNumber(paradexOrder.amount); - tokenOrder.source = source; + tokenOrder.source = PARADEX_SOURCE; tokenOrder.observedTimestamp = observedTimestamp; tokenOrder.orderType = orderType; tokenOrder.price = price; @@ -62,5 +58,6 @@ export function parseParadexOrder( tokenOrder.quoteAssetSymbol = paradexMarket.quoteToken; tokenOrder.quoteAssetAddress = paradexMarket.quoteTokenAddress as string; tokenOrder.quoteVolume = price.times(amount); + tokenOrder.makerAddress = 'unknown'; return tokenOrder; } diff --git a/packages/pipeline/src/parsers/radar_orders/index.ts b/packages/pipeline/src/parsers/radar_orders/index.ts new file mode 100644 index 0000000000..6ecedc0c36 --- /dev/null +++ b/packages/pipeline/src/parsers/radar_orders/index.ts @@ -0,0 +1,118 @@ +import { ObjectMap } from '@0x/types'; +import { BigNumber } from '@0x/utils'; +import { RadarBook, RadarMarket, RadarSignedOrder } from '@radarrelay/types'; +import * as R from 'ramda'; + +import { aggregateOrders, GenericRawOrder } from '../utils'; + +import { RADAR_SOURCE } from '../../data_sources/radar'; +import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; +import { OrderType } from '../../types'; + +export interface AggregateOrdersByMaker { + makerAddress: string; + price: string; + amount: BigNumber; +} + +/** + * Marquee function of this file. + * 1) Takes in orders from an orderbook, + * other information attached. + * @param radarOrderbook A raw orderbook that we pull from the radar API. + * @param radarMarket An object containing market data also directly from the API. + * @param observedTimestamp Time at which the orders for the market were pulled. + */ +export function parseRadarOrders( + radarOrderbook: RadarBook, + radarMarket: RadarMarket, + observedTimestamp: number, +): TokenOrder[] { + const aggregatedBids = _aggregateOrdersByMaker(radarMarket, radarOrderbook.bids); + const aggregatedAsks = _aggregateOrdersByMaker(radarMarket, radarOrderbook.asks); + const parsedBids = aggregatedBids.map(order => + parseRadarOrder(radarMarket, observedTimestamp, OrderType.Bid, order), + ); + const parsedAsks = aggregatedAsks.map(order => + parseRadarOrder(radarMarket, observedTimestamp, OrderType.Ask, order), + ); + return parsedBids.concat(parsedAsks); +} + +/** + * Parse a single aggregated radar order in order to form a tokenOrder entity + * which can be saved into the database. + * @param radarMarket An object containing information about the market where these + * trades have been placed. + * @param observedTimestamp The time when the API response returned back to us. + * @param orderType 'bid' or 'ask' enum. + * @param aggregateOrder An AggregateOrdersByMaker instance which we will convert to volume-basis. + */ +export function parseRadarOrder( + radarMarket: RadarMarket, + observedTimestamp: number, + orderType: OrderType, + aggregateOrder: AggregateOrdersByMaker, +): TokenOrder { + const tokenOrder = new TokenOrder(); + const price = new BigNumber(aggregateOrder.price); + const amount = aggregateOrder.amount; + const splitId = radarMarket.id.split('-'); + + tokenOrder.source = RADAR_SOURCE; + tokenOrder.observedTimestamp = observedTimestamp; + tokenOrder.orderType = orderType; + tokenOrder.price = price; + + tokenOrder.baseAssetSymbol = splitId[0]; + tokenOrder.baseAssetAddress = radarMarket.baseTokenAddress || null; + tokenOrder.baseVolume = amount; + + tokenOrder.quoteAssetSymbol = splitId[1]; + tokenOrder.quoteAssetAddress = radarMarket.quoteTokenAddress || null; + tokenOrder.quoteVolume = price.times(amount); + + tokenOrder.makerAddress = aggregateOrder.makerAddress; + return tokenOrder; +} + +function _toGeneric(radarMarket: RadarMarket, radarOrder: RadarSignedOrder): GenericRawOrder | undefined { + if (radarMarket.baseTokenDecimals === undefined) { + return undefined; + } + return { + price: radarOrder.price.toString(), + // Use the remaining fillable amount + amount: radarOrder.remainingBaseTokenAmount.toString(), + }; +} + +function _aggregateOrdersByMaker(radarMarket: RadarMarket, radarOrders: RadarSignedOrder[]): AggregateOrdersByMaker[] { + // group all orders by their maker + const ordersByMaker: ObjectMap = radarOrders.reduce( + (acc: ObjectMap, val: RadarSignedOrder) => { + const makerAddress = val.signedOrder.makerAddress; + if (acc[makerAddress]) { + acc[makerAddress].push(val); + } else { + acc[makerAddress] = []; + } + return acc; + }, + {}, + ); + const transformToGeneric = (radarOrder: RadarSignedOrder) => _toGeneric(radarMarket, radarOrder); + const aggregationTuples: AggregateOrdersByMaker[][] = (R.keys(ordersByMaker) as string[]).map((maker: string) => { + const generalizedOrders = _removeUndefined(R.map(transformToGeneric, ordersByMaker[maker])); + const aggregatedOrders = aggregateOrders(generalizedOrders); + return aggregatedOrders.map((order: [string, BigNumber]) => ({ + makerAddress: maker, + price: order[0], + amount: order[1], + })); + }); + return R.unnest(aggregationTuples); +} + +// tslint:disable-next-line:no-unbound-method +const _removeUndefined = R.reject(R.isNil); diff --git a/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts index 4e00f258f3..96d82be5a2 100644 --- a/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts +++ b/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts @@ -2,7 +2,7 @@ import { logUtils } from '@0x/utils'; import * as R from 'ramda'; import { Connection, ConnectionOptions, createConnection } from 'typeorm'; -import { DDEX_SOURCE, DdexMarket, DdexSource } from '../data_sources/ddex'; +import { DdexMarket, DdexSource } from '../data_sources/ddex'; import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; import * as ormConfig from '../ormconfig'; import { parseDdexOrders } from '../parsers/ddex_orders'; @@ -43,7 +43,7 @@ async function getAndSaveMarketOrderbookAsync(ddexSource: DdexSource, market: Dd const observedTimestamp = Date.now(); logUtils.log(`${market.id}: Parsing orders.`); - const orders = parseDdexOrders(orderBook, market, observedTimestamp, DDEX_SOURCE); + const orders = parseDdexOrders(orderBook, market, observedTimestamp); if (orders.length > 0) { logUtils.log(`${market.id}: Saving ${orders.length} orders.`); diff --git a/packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts index 490b17766c..42e9fa5f09 100644 --- a/packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts +++ b/packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts @@ -2,7 +2,7 @@ import { logUtils } from '@0x/utils'; import * as R from 'ramda'; import { Connection, ConnectionOptions, createConnection } from 'typeorm'; -import { IDEX_SOURCE, IdexSource } from '../data_sources/idex'; +import { IdexSource } from '../data_sources/idex'; import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; import * as ormConfig from '../ormconfig'; import { parseIdexOrders } from '../parsers/idex_orders'; @@ -51,7 +51,7 @@ async function getAndSaveMarketOrderbookAsync(idexSource: IdexSource, marketId: } logUtils.log(`${marketId}: Parsing orders.`); - const orders = parseIdexOrders(orderBook, observedTimestamp, IDEX_SOURCE); + const orders = parseIdexOrders(orderBook, observedTimestamp); if (orders.length > 0) { logUtils.log(`${marketId}: Saving ${orders.length} orders.`); diff --git a/packages/pipeline/src/scripts/pull_nonfungible_dot_com_trades.ts b/packages/pipeline/src/scripts/pull_nonfungible_dot_com_trades.ts new file mode 100644 index 0000000000..8d563400cd --- /dev/null +++ b/packages/pipeline/src/scripts/pull_nonfungible_dot_com_trades.ts @@ -0,0 +1,43 @@ +// tslint:disable:no-console +import 'reflect-metadata'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { getTradesAsync, knownPublishers } from '../data_sources/nonfungible_dot_com'; +import { NonfungibleDotComTrade } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseNonFungibleDotComTrades } from '../parsers/nonfungible_dot_com'; +import { handleError } from '../utils'; + +// Number of trades to save at once. +const BATCH_SAVE_SIZE = 1000; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + await getAndSaveTradesAsync(); + process.exit(0); +})().catch(handleError); + +async function getAndSaveTradesAsync(): Promise { + const tradesRepository = connection.getRepository(NonfungibleDotComTrade); + + for (const publisher of knownPublishers) { + console.log(`Getting latest trades for NFT ${publisher}...`); + const tradeWithHighestBlockNumber = await tradesRepository + .createQueryBuilder('nonfungible_dot_com_trades') + .where('nonfungible_dot_com_trades.publisher = :publisher', { publisher }) + .orderBy({ 'nonfungible_dot_com_trades.block_number': 'DESC' }) + .getOne(); + const highestExistingBlockNumber = + tradeWithHighestBlockNumber === undefined ? 0 : tradeWithHighestBlockNumber.blockNumber; + console.log(`Highest block number in existing trades: ${highestExistingBlockNumber}`); + const rawTrades = await getTradesAsync(publisher, highestExistingBlockNumber); + console.log(`Parsing ${rawTrades.length} trades...`); + const trades = parseNonFungibleDotComTrades(rawTrades, publisher); + console.log(`Saving ${rawTrades.length} trades...`); + await tradesRepository.save(trades, { chunk: Math.ceil(trades.length / BATCH_SAVE_SIZE) }); + } + const newTotalTrades = await tradesRepository.count(); + console.log(`Done saving trades. There are now ${newTotalTrades} total NFT trades.`); +} diff --git a/packages/pipeline/src/scripts/pull_oasis_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_oasis_orderbook_snapshots.ts index c4dcf6c83d..c3c286a0f9 100644 --- a/packages/pipeline/src/scripts/pull_oasis_orderbook_snapshots.ts +++ b/packages/pipeline/src/scripts/pull_oasis_orderbook_snapshots.ts @@ -2,7 +2,7 @@ import { logUtils } from '@0x/utils'; import * as R from 'ramda'; import { Connection, ConnectionOptions, createConnection } from 'typeorm'; -import { OASIS_SOURCE, OasisMarket, OasisSource } from '../data_sources/oasis'; +import { OasisMarket, OasisSource } from '../data_sources/oasis'; import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; import * as ormConfig from '../ormconfig'; import { parseOasisOrders } from '../parsers/oasis_orders'; @@ -46,7 +46,7 @@ async function getAndSaveMarketOrderbookAsync(oasisSource: OasisSource, market: const observedTimestamp = Date.now(); logUtils.log(`${market.id}: Parsing orders.`); - const orders = parseOasisOrders(orderBook, market, observedTimestamp, OASIS_SOURCE); + const orders = parseOasisOrders(orderBook, market, observedTimestamp); if (orders.length > 0) { logUtils.log(`${market.id}: Saving ${orders.length} orders.`); diff --git a/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts index 34345f3557..7fc565ca04 100644 --- a/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts +++ b/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts @@ -2,7 +2,6 @@ import { logUtils } from '@0x/utils'; import { Connection, ConnectionOptions, createConnection } from 'typeorm'; import { - PARADEX_SOURCE, ParadexActiveMarketsResponse, ParadexMarket, ParadexSource, @@ -75,7 +74,7 @@ async function getAndSaveMarketOrderbookAsync(paradexSource: ParadexSource, mark const observedTimestamp = Date.now(); logUtils.log(`${market.symbol}: Parsing orders.`); - const orders = parseParadexOrders(paradexOrderbookResponse, market, observedTimestamp, PARADEX_SOURCE); + const orders = parseParadexOrders(paradexOrderbookResponse, market, observedTimestamp); if (orders.length > 0) { logUtils.log(`${market.symbol}: Saving ${orders.length} orders.`); diff --git a/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts new file mode 100644 index 0000000000..18f5a54b20 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts @@ -0,0 +1,56 @@ +import { logUtils } from '@0x/utils'; +import { RadarMarket } from '@radarrelay/types'; +import * as R from 'ramda'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { RadarSource } from '../data_sources/radar'; +import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseRadarOrders } from '../parsers/radar_orders'; +import { handleError } from '../utils'; + +// Number of orders to save at once. +const BATCH_SAVE_SIZE = 1000; + +// Number of markets to retrieve orderbooks for at once. +const MARKET_ORDERBOOK_REQUEST_BATCH_SIZE = 50; + +// Delay between market orderbook requests. +const MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY = 5000; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + const radarSource = new RadarSource(); + const markets = await radarSource.getActiveMarketsAsync(); + for (const marketsChunk of R.splitEvery(MARKET_ORDERBOOK_REQUEST_BATCH_SIZE, markets)) { + await Promise.all( + marketsChunk.map(async (market: RadarMarket) => getAndSaveMarketOrderbookAsync(radarSource, market)), + ); + await new Promise(resolve => setTimeout(resolve, MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY)); + } + process.exit(0); +})().catch(handleError); + +/** + * Retrieve orderbook from radar API for a given market. Parse orders and insert + * them into our database. + * @param radarSource Data source which can query radar API. + * @param market Object from radar API containing market data. + */ +async function getAndSaveMarketOrderbookAsync(radarSource: RadarSource, market: RadarMarket): Promise { + const orderBook = await radarSource.getMarketOrderbookAsync(market.id); + const observedTimestamp = Date.now(); + + logUtils.log(`${market.id}: Parsing orders.`); + const orders = parseRadarOrders(orderBook, market, observedTimestamp); + + if (orders.length > 0) { + logUtils.log(`${market.id}: Saving ${orders.length} orders.`); + const TokenOrderRepository = connection.getRepository(TokenOrder); + await TokenOrderRepository.save(orders, { chunk: Math.ceil(orders.length / BATCH_SAVE_SIZE) }); + } else { + logUtils.log(`${market.id}: 0 orders to save.`); + } +} diff --git a/packages/pipeline/src/scripts/pull_slippage.ts b/packages/pipeline/src/scripts/pull_slippage.ts new file mode 100644 index 0000000000..6d308ee34a --- /dev/null +++ b/packages/pipeline/src/scripts/pull_slippage.ts @@ -0,0 +1,64 @@ +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { logUtils } from '@0x/utils'; + +import { EdpsSource } from '../data_sources/dex_prices'; +import { CryptoCompareOHLCVSource } from '../data_sources/ohlcv_external/crypto_compare'; +import { Slippage } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { calculateSlippage } from '../transformers/slippage'; +import { handleError } from '../utils'; + +// Number of orders to save at once. +const BATCH_SAVE_SIZE = 1000; + +// Maximum requests per second to CryptoCompare +const CRYPTO_COMPARE_MAX_REQS_PER_SECOND = 60; + +// USD amounts for slippage depths +// tslint:disable-next-line:custom-no-magic-numbers +const USD_AMOUNTS = [10, 100, 1000, 10000]; + +// TODO: fetch from database +const TOKENS = ['BAT', 'DAI', 'FUN', 'MANA', 'OMG', 'REP', 'TUSD', 'ZRX', 'MKR', 'BNB', 'USDC', 'LOOM', 'DNT', 'CVC']; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + const edpsSource = new EdpsSource(); + const cryptoCompareSource = new CryptoCompareOHLCVSource(CRYPTO_COMPARE_MAX_REQS_PER_SECOND); + + logUtils.log('Fetching slippage records'); + const nestedSlippages: Slippage[][][] = await Promise.all( + TOKENS.map(async symbol => { + const usdPrice = await cryptoCompareSource.getUsdPriceAsync(symbol); + return Promise.all( + USD_AMOUNTS.map(async usdAmount => { + const amount = usdAmount / usdPrice; + try { + const buyEdps = await edpsSource.getEdpsAsync('buy', symbol, amount); + const sellEdps = await edpsSource.getEdpsAsync('sell', symbol, amount); + return Object.keys(buyEdps).map(exchange => { + const slippage: Slippage = calculateSlippage(usdAmount, exchange, buyEdps, sellEdps); + return slippage; + }); + } catch (e) { + logUtils.log(`Error getting data for symbol=${symbol}, amount=${amount}`); + logUtils.log(e); + return [new Slippage()]; + } + }), + ); + }), + ); + const slippagesWithEmptyRecords = nestedSlippages + .reduce((acc, val) => acc.concat(val)) + .reduce((acc, val) => acc.concat(val)); + const slippages = slippagesWithEmptyRecords.filter(slippage => slippage.observedTimestamp); + const SlippageRepository = connection.getRepository(Slippage); + logUtils.log(`Saving ${slippages.length} records to database`); + await SlippageRepository.save(slippages, { chunk: Math.ceil(slippages.length / BATCH_SAVE_SIZE) }); + logUtils.log('Done'); + process.exit(0); +})().catch(handleError); diff --git a/packages/pipeline/src/transformers/slippage/index.ts b/packages/pipeline/src/transformers/slippage/index.ts new file mode 100644 index 0000000000..38e2e6f4a8 --- /dev/null +++ b/packages/pipeline/src/transformers/slippage/index.ts @@ -0,0 +1,36 @@ +import { BigNumber } from '@0x/utils'; + +import { EdpsWrapper } from '../../data_sources/dex_prices'; +import { Slippage } from '../../entities'; + +/** + * Calculates slippage and returns Slippage entity. + * + * @param usdAmount Amount to buy/sell in USD. + * @param exchange Exchange where we are calculating slippage. + * @param buyEdps Ethereum DEX price service object for buy side. + * @param sellEdps Ethereum DEX price service object for sell side. + * + */ + +export function calculateSlippage( + usdAmount: number, + exchange: string, + buyEdps: EdpsWrapper, + sellEdps: EdpsWrapper, +): Slippage { + const b = buyEdps[exchange]; + const s = sellEdps[exchange]; + const slippage = new Slippage(); + if (b && s && b.avgPrice && s.avgPrice) { + slippage.observedTimestamp = b.timestamp; + slippage.symbol = b.tokenSymbol; + slippage.exchange = exchange; + slippage.usdAmount = new BigNumber(usdAmount); + slippage.tokenAmount = new BigNumber(Number(b.tokenAmount)); // API returns a string + slippage.avgPriceInEthBuy = new BigNumber(b.avgPrice); + slippage.avgPriceInEthSell = new BigNumber(s.avgPrice); + slippage.slippage = new BigNumber((b.avgPrice - s.avgPrice) / b.avgPrice); + } + return slippage; +} diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts index 094c0178e6..0342481e0e 100644 --- a/packages/pipeline/src/utils/index.ts +++ b/packages/pipeline/src/utils/index.ts @@ -1,4 +1,4 @@ -import { BigNumber } from '@0x/utils'; +import { BigNumber, fetchAsync } from '@0x/utils'; export * from './transformers'; export * from './constants'; @@ -51,3 +51,16 @@ export function handleError(e: any): void { } process.exit(1); } + +/** + * Does fetchAsync(), and checks the status code, throwing if it doesn't indicate success. + */ +export async function fetchSuccessfullyOrThrowAsync(url: string): Promise { + const response = await fetchAsync(url); + if (!response.ok) { + throw new Error( + `Failed to fetch URL ${url}. Unsuccessful HTTP status code (${response.status}): ${response.statusText}`, + ); + } + return response.json(); +} diff --git a/packages/pipeline/test/data_sources/radar/index_test.ts b/packages/pipeline/test/data_sources/radar/index_test.ts new file mode 100644 index 0000000000..8d82ff6bf5 --- /dev/null +++ b/packages/pipeline/test/data_sources/radar/index_test.ts @@ -0,0 +1,76 @@ +import { BigNumber } from '@0x/utils'; +import { RadarOrderState, RadarOrderType } from '@radarrelay/types'; +import * as chai from 'chai'; +import 'mocha'; + +import { RadarSource } from '../../../src/data_sources/radar'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +const rawResponse = { + orderHash: '0x60bc235f7887a50801c8fc1fc18fb0625ac5f3962cdc1bd59567a6929db8b2ec', + type: 'BID', + state: 'OPEN', + baseTokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + quoteTokenAddress: '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359', + remainingBaseTokenAmount: '9.079731811797989766', + remainingQuoteTokenAmount: '1099.999999999999999889', + price: '121.14895272244560081697', + createdDate: '2019-02-13 21:35:53', + signedOrder: { + exchangeAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + senderAddress: '0x0000000000000000000000000000000000000000', + makerAddress: '0x56178a0d5f301baf6cf3e1cd53d9863437345bf9', + takerAddress: '0x0000000000000000000000000000000000000000', + makerAssetData: '0xf47261b000000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359', + takerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + feeRecipientAddress: '0xa258b39954cef5cb142fd567a46cddb31a670124', + makerAssetAmount: '1099999999999999999889', + takerAssetAmount: '9079731811797989766', + makerFee: '0', + takerFee: '0', + expirationTimeSeconds: '1550094353', + signature: + '0x1ce161d02ad63fe7308e9cd5e97583a8873331d1b72d90e9f3863d9fcba2518cb91ab2fe7de94e4afb39742acdc820abbff2dc0622c8d3865917fade62f16322ae03', + salt: '1550093753237', + }, +}; + +const parsedResponse = { + orderHash: '0x60bc235f7887a50801c8fc1fc18fb0625ac5f3962cdc1bd59567a6929db8b2ec', + type: 'BID' as RadarOrderType, + state: 'OPEN' as RadarOrderState, + baseTokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + quoteTokenAddress: '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359', + remainingBaseTokenAmount: new BigNumber('9.079731811797989766'), + remainingQuoteTokenAmount: new BigNumber('1099.999999999999999889'), + price: new BigNumber('121.14895272244560081697'), + createdDate: '2019-02-13 21:35:53', + signedOrder: { + exchangeAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + senderAddress: '0x0000000000000000000000000000000000000000', + makerAddress: '0x56178a0d5f301baf6cf3e1cd53d9863437345bf9', + takerAddress: '0x0000000000000000000000000000000000000000', + makerAssetData: '0xf47261b000000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359', + takerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + feeRecipientAddress: '0xa258b39954cef5cb142fd567a46cddb31a670124', + makerAssetAmount: new BigNumber('1099999999999999999889'), + takerAssetAmount: new BigNumber('9079731811797989766'), + makerFee: new BigNumber('0'), + takerFee: new BigNumber('0'), + expirationTimeSeconds: new BigNumber('1550094353'), + signature: + '0x1ce161d02ad63fe7308e9cd5e97583a8873331d1b72d90e9f3863d9fcba2518cb91ab2fe7de94e4afb39742acdc820abbff2dc0622c8d3865917fade62f16322ae03', + salt: new BigNumber('1550093753237'), + }, +}; + +describe('RadarSource', () => { + describe('parseRadarOrderResponse', () => { + it('Correctly parses a Radar orderbook response to a RadarBook', () => { + expect(RadarSource.parseRadarOrderResponse(rawResponse)).deep.equal(parsedResponse); + }); + }); +}); diff --git a/packages/pipeline/test/entities/nft_trades_test.ts b/packages/pipeline/test/entities/nft_trades_test.ts new file mode 100644 index 0000000000..01571e8f71 --- /dev/null +++ b/packages/pipeline/test/entities/nft_trades_test.ts @@ -0,0 +1,48 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; +import 'reflect-metadata'; + +import { NonfungibleDotComTrade } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const baseTrade: NonfungibleDotComTrade = { + assetDescriptor: 'Kitty #1002', + assetId: '1002', + blockNumber: 4608542, + blockTimestamp: 1543544083704, + buyerAddress: '0x316c55d1895a085c4b39a98ecb563f509301aaf7', + logIndex: 28, + marketAddress: '0xb1690C08E213a35Ed9bAb7B318DE14420FB57d8C', + meta: { + cattribute_body: 'munchkin', + cattribute_coloreyes: 'mintgreen', + cattribute_colorprimary: 'orangesoda', + cattribute_colorsecondary: 'coffee', + cattribute_colortertiary: 'kittencream', + cattribute_eyes: 'thicccbrowz', + cattribute_mouth: 'soserious', + cattribute_pattern: 'totesbasic', + generation: '0', + is_exclusive: false, + is_fancy: false, + }, + sellerAddress: '0xba52c75764d6f594735dc735be7f1830cdf58ddf', + totalPrice: new BigNumber('9751388888888889'), + transactionHash: '0x468168419be7e442d5ff32d264fab24087b744bc2e37fdbac7024e1e74f4c6c8', + usdPrice: new BigNumber('3.71957'), + publisher: 'cryptokitties', +}; + +// tslint:disable:custom-no-magic-numbers +describe('NonfungibleDotComTrade entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const tradesRepository = connection.getRepository(NonfungibleDotComTrade); + await testSaveAndFindEntityAsync(tradesRepository, baseTrade); + }); +}); diff --git a/packages/pipeline/test/entities/slippage_test.ts b/packages/pipeline/test/entities/slippage_test.ts new file mode 100644 index 0000000000..0dc655adeb --- /dev/null +++ b/packages/pipeline/test/entities/slippage_test.ts @@ -0,0 +1,34 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; +import 'reflect-metadata'; + +import { Slippage } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const slippage = { + observedTimestamp: 1549587475793, + symbol: 'ZRX', + exchange: 'Radar Relay', + usdAmount: new BigNumber(10), + tokenAmount: new BigNumber(25), + avgPriceInEthBuy: new BigNumber(0.0022), + avgPriceInEthSell: new BigNumber(0.002), + slippage: new BigNumber(0.01), +}; + +// tslint:disable:custom-no-magic-numbers +describe('Slippage entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const slippages = [slippage]; + const slippageRepository = connection.getRepository(Slippage); + for (const slippageRecord of slippages) { + await testSaveAndFindEntityAsync(slippageRepository, slippageRecord); + } + }); +}); diff --git a/packages/pipeline/test/entities/token_order_test.ts b/packages/pipeline/test/entities/token_order_test.ts index c6057f5aae..8f2df569b0 100644 --- a/packages/pipeline/test/entities/token_order_test.ts +++ b/packages/pipeline/test/entities/token_order_test.ts @@ -20,6 +20,7 @@ const tokenOrderbookSnapshot: TokenOrderbookSnapshot = { quoteAssetSymbol: 'ABC', quoteAssetAddress: '0x00923b9a074762b93650716333b3e1473a15048e', quoteVolume: new BigNumber(12.3234234), + makerAddress: 'unknown', }; describe('TokenOrderbookSnapshot entity', () => { diff --git a/packages/pipeline/test/parsers/ddex_orders/index_test.ts b/packages/pipeline/test/parsers/ddex_orders/index_test.ts index d6f69e090e..de71c9c26f 100644 --- a/packages/pipeline/test/parsers/ddex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/ddex_orders/index_test.ts @@ -31,7 +31,6 @@ describe('ddex_orders', () => { }; const observedTimestamp: number = Date.now(); const orderType: OrderType = OrderType.Bid; - const source: string = 'ddex'; const expected = new TokenOrder(); expected.source = 'ddex'; @@ -44,8 +43,8 @@ describe('ddex_orders', () => { expected.baseAssetSymbol = 'DEF'; expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; expected.baseVolume = new BigNumber(10); - - const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, source, ddexOrder); + expected.makerAddress = 'unknown'; + const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, ddexOrder); expect(actual).deep.equal(expected); }); }); diff --git a/packages/pipeline/test/parsers/idex_orders/index_test.ts b/packages/pipeline/test/parsers/idex_orders/index_test.ts index 48b019732a..7ba3adbaea 100644 --- a/packages/pipeline/test/parsers/idex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/idex_orders/index_test.ts @@ -32,7 +32,6 @@ describe('idex_orders', () => { }; const observedTimestamp: number = Date.now(); const orderType: OrderType = OrderType.Bid; - const source: string = 'idex'; const expected = new TokenOrder(); expected.source = 'idex'; @@ -45,8 +44,8 @@ describe('idex_orders', () => { expected.quoteAssetSymbol = 'DEF'; expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; expected.quoteVolume = new BigNumber(5); - - const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, source, idexOrder); + expected.makerAddress = 'unknown'; + const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, idexOrder); expect(actual).deep.equal(expected); }); it('correctly converts ask type idexOrder to TokenOrder entity', () => { @@ -66,7 +65,6 @@ describe('idex_orders', () => { }; const observedTimestamp: number = Date.now(); const orderType: OrderType = OrderType.Ask; - const source: string = 'idex'; const expected = new TokenOrder(); expected.source = 'idex'; @@ -79,8 +77,8 @@ describe('idex_orders', () => { expected.quoteAssetSymbol = 'DEF'; expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; expected.quoteVolume = new BigNumber(5); - - const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, source, idexOrder); + expected.makerAddress = 'unknown'; + const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, idexOrder); expect(actual).deep.equal(expected); }); }); diff --git a/packages/pipeline/test/parsers/nonfungible_dot_com/index_test.ts b/packages/pipeline/test/parsers/nonfungible_dot_com/index_test.ts new file mode 100644 index 0000000000..f7929a5460 --- /dev/null +++ b/packages/pipeline/test/parsers/nonfungible_dot_com/index_test.ts @@ -0,0 +1,86 @@ +// tslint:disable:custom-no-magic-numbers +import { BigNumber } from '@0x/utils'; +import * as chai from 'chai'; +import 'mocha'; + +import { NonfungibleDotComTradeResponse } from '../../../src/data_sources/nonfungible_dot_com'; +import { NonfungibleDotComTrade } from '../../../src/entities'; +import { _parseNonFungibleDotComTrade } from '../../../src/parsers/nonfungible_dot_com'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +const input: NonfungibleDotComTradeResponse = { + _id: '5b4cd04244abdb5ac3a8063f', + assetDescriptor: 'Kitty #1002', + assetId: '1002', + blockNumber: 4608542, + blockTimestamp: '2017-11-23T18:50:19.000Z', + buyer: '0x316c55d1895a085c4b39a98ecb563f509301aaf7', + logIndex: 28, + nftAddress: '0xb1690C08E213a35Ed9bAb7B318DE14420FB57d8C', + marketAddress: '0xb1690C08E213a35Ed9bAb7B318DE14420FB57d8C', + tokenTicker: 'eth', + meta: { + cattribute_body: 'munchkin', + cattribute_coloreyes: 'mintgreen', + cattribute_colorprimary: 'orangesoda', + cattribute_colorsecondary: 'coffee', + cattribute_colortertiary: 'kittencream', + cattribute_eyes: 'thicccbrowz', + cattribute_mouth: 'soserious', + cattribute_pattern: 'totesbasic', + generation: '0', + is_exclusive: false, + is_fancy: false, + }, + seller: '0xba52c75764d6f594735dc735be7f1830cdf58ddf', + totalDecimalPrice: 0.00975138888888889, + totalPrice: '9751388888888889', + transactionHash: '0x468168419be7e442d5ff32d264fab24087b744bc2e37fdbac7024e1e74f4c6c8', + usdPrice: 3.71957, + currencyTransfer: {}, + image: '', + composedOf: '', + asset_link: '', + seller_address_link: '', + buyer_address_link: '', +}; + +const expected: NonfungibleDotComTrade = { + assetDescriptor: 'Kitty #1002', + assetId: '1002', + blockNumber: 4608542, + blockTimestamp: 1511463019000, + buyerAddress: '0x316c55d1895a085c4b39a98ecb563f509301aaf7', + logIndex: 28, + marketAddress: '0xb1690C08E213a35Ed9bAb7B318DE14420FB57d8C', + meta: { + cattribute_body: 'munchkin', + cattribute_coloreyes: 'mintgreen', + cattribute_colorprimary: 'orangesoda', + cattribute_colorsecondary: 'coffee', + cattribute_colortertiary: 'kittencream', + cattribute_eyes: 'thicccbrowz', + cattribute_mouth: 'soserious', + cattribute_pattern: 'totesbasic', + generation: '0', + is_exclusive: false, + is_fancy: false, + }, + sellerAddress: '0xba52c75764d6f594735dc735be7f1830cdf58ddf', + totalPrice: new BigNumber('9751388888888889'), + transactionHash: '0x468168419be7e442d5ff32d264fab24087b744bc2e37fdbac7024e1e74f4c6c8', + usdPrice: new BigNumber('3.71957'), + publisher: 'cryptokitties', +}; + +describe('nonfungible.com', () => { + describe('_parseNonFungibleDotComTrade', () => { + it(`converts NonfungibleDotComTradeResponse to NonfungibleDotComTrade entity`, () => { + const actual = _parseNonFungibleDotComTrade(expected.publisher, input); + expect(actual).deep.equal(expected); + }); + }); +}); diff --git a/packages/pipeline/test/parsers/oasis_orders/index_test.ts b/packages/pipeline/test/parsers/oasis_orders/index_test.ts index 401fedff87..08ff1ef450 100644 --- a/packages/pipeline/test/parsers/oasis_orders/index_test.ts +++ b/packages/pipeline/test/parsers/oasis_orders/index_test.ts @@ -28,7 +28,6 @@ describe('oasis_orders', () => { }; const observedTimestamp: number = Date.now(); const orderType: OrderType = OrderType.Bid; - const source: string = 'oasis'; const expected = new TokenOrder(); expected.source = 'oasis'; @@ -41,8 +40,8 @@ describe('oasis_orders', () => { expected.quoteAssetSymbol = 'ABC'; expected.quoteAssetAddress = null; expected.quoteVolume = new BigNumber(5); - - const actual = parseOasisOrder(oasisMarket, observedTimestamp, orderType, source, oasisOrder); + expected.makerAddress = 'unknown'; + const actual = parseOasisOrder(oasisMarket, observedTimestamp, orderType, oasisOrder); expect(actual).deep.equal(expected); }); }); diff --git a/packages/pipeline/test/parsers/paradex_orders/index_test.ts b/packages/pipeline/test/parsers/paradex_orders/index_test.ts index c5dd8751b2..5379153387 100644 --- a/packages/pipeline/test/parsers/paradex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/paradex_orders/index_test.ts @@ -33,7 +33,6 @@ describe('paradex_orders', () => { }; const observedTimestamp: number = Date.now(); const orderType: OrderType = OrderType.Bid; - const source: string = 'paradex'; const expected = new TokenOrder(); expected.source = 'paradex'; @@ -46,8 +45,8 @@ describe('paradex_orders', () => { expected.quoteAssetSymbol = 'ABC'; expected.quoteAssetAddress = '0x0000000000000000000000000000000000000000'; expected.quoteVolume = new BigNumber(412 * 0.1245); - - const actual = parseParadexOrder(paradexMarket, observedTimestamp, orderType, source, paradexOrder); + expected.makerAddress = 'unknown'; + const actual = parseParadexOrder(paradexMarket, observedTimestamp, orderType, paradexOrder); expect(actual).deep.equal(expected); }); }); diff --git a/packages/pipeline/test/parsers/radar_orders/index_test.ts b/packages/pipeline/test/parsers/radar_orders/index_test.ts new file mode 100644 index 0000000000..29ae41a00d --- /dev/null +++ b/packages/pipeline/test/parsers/radar_orders/index_test.ts @@ -0,0 +1,55 @@ +import { BigNumber } from '@0x/utils'; +import { RadarMarket } from '@radarrelay/types'; +import * as chai from 'chai'; +import 'mocha'; + +import { TokenOrderbookSnapshot as TokenOrder } from '../../../src/entities'; +import { AggregateOrdersByMaker, parseRadarOrder } from '../../../src/parsers/radar_orders'; +import { OrderType } from '../../../src/types'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('radar_orders', () => { + describe('parseRadarOrder', () => { + it('converts radarOrder to TokenOrder entity', () => { + const radarOrder: AggregateOrdersByMaker = { + makerAddress: '0x6eC92694ea172ebC430C30fa31De87620967A082', + price: '0.01', + amount: new BigNumber(10000000000), + }; + const radarMarket = ({ + id: 'WETH-DAI', + displayName: 'WETH/DAI', + baseTokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + quoteTokenAddress: '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359', + baseTokenDecimals: 18, + quoteTokenDecimals: 18, + quoteIncrement: 8, + minOrderSize: new BigNumber('0.00692535'), + maxOrderSize: new BigNumber('1000000000'), + score: 99.66, + // Radar types are defined using an older version of BigNumber, so need to be force cast. + } as any) as RadarMarket; + const observedTimestamp: number = Date.now(); + const orderType: OrderType = OrderType.Bid; + + const expected = new TokenOrder(); + expected.source = 'radar'; + expected.observedTimestamp = observedTimestamp; + expected.orderType = OrderType.Bid; + expected.price = new BigNumber(0.01); + expected.quoteAssetSymbol = 'DAI'; + expected.quoteAssetAddress = '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359'; + expected.quoteVolume = new BigNumber(100000000); + expected.baseAssetSymbol = 'WETH'; + expected.baseAssetAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; + expected.baseVolume = new BigNumber(10000000000); + expected.makerAddress = '0x6eC92694ea172ebC430C30fa31De87620967A082'; + const actual = parseRadarOrder(radarMarket, observedTimestamp, orderType, radarOrder); + expect(actual).deep.equal(expected); + }); + }); +}); diff --git a/packages/pipeline/test/transformers/slippage/index_test.ts b/packages/pipeline/test/transformers/slippage/index_test.ts new file mode 100644 index 0000000000..8528051215 --- /dev/null +++ b/packages/pipeline/test/transformers/slippage/index_test.ts @@ -0,0 +1,61 @@ +import { BigNumber } from '@0x/utils'; +import * as chai from 'chai'; +import 'mocha'; + +import { EdpsWrapper } from '../../../src/data_sources/dex_prices'; +import { Slippage } from '../../../src/entities'; +import { calculateSlippage } from '../../../src/transformers/slippage'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('slippage', () => { + describe('calculateSlippage', () => { + it('calculates slippage correctly', () => { + const exchange = 'Radar Relay'; + const ts = 1549961441473; + const symbol = 'DAI'; + const amount = 1000; + const buyPrice = 10; + const sellPrice = 9; + const expectedSlippage = 0.1; + + const buyEdps: EdpsWrapper = {}; + buyEdps[exchange] = { + exchangeName: exchange, + totalPrice: buyPrice, + tokenAmount: amount, + tokenSymbol: symbol, + avgPrice: buyPrice / amount, + timestamp: ts, + error: '', + }; + + const sellEdps: EdpsWrapper = {}; + sellEdps[exchange] = { + exchangeName: exchange, + totalPrice: sellPrice, + tokenAmount: amount, + tokenSymbol: symbol, + avgPrice: sellPrice / amount, + timestamp: ts, + error: '', + }; + const expected = new Slippage(); + expected.observedTimestamp = ts; + expected.symbol = symbol; + expected.exchange = exchange; + expected.usdAmount = new BigNumber(amount); + expected.tokenAmount = new BigNumber(amount); // API returns a string + expected.avgPriceInEthBuy = new BigNumber(buyPrice / amount); + expected.avgPriceInEthSell = new BigNumber(sellPrice / amount); + expected.slippage = new BigNumber(0.1); + + const actual = calculateSlippage(amount, exchange, buyEdps, sellEdps); + const actualSlippage: BigNumber = actual.slippage ? actual.slippage : new BigNumber(0); + expect(actualSlippage.toNumber()).to.be.closeTo(expectedSlippage, 0.0001); + }); + }); +}); diff --git a/packages/website/ts/pages/governance/address_table.tsx b/packages/website/ts/pages/governance/address_table.tsx index f861a3aee8..28bfd09687 100644 --- a/packages/website/ts/pages/governance/address_table.tsx +++ b/packages/website/ts/pages/governance/address_table.tsx @@ -68,15 +68,15 @@ export class AddressTable extends React.Component` - background - color: #fff; - border -radius;: 4;px; - Margin-bottom;: $;{props => props.marginBottom || '25px';} - Padding: 10;px; 30;px; - Height: 230;px; - Overflow - y;: auto; + background-color: #fff; + border-radius: 4px; + margin-bottom: ${props => props.marginBottom || '25px'}; + padding: 10px 30px; + height: 230px; + overflow-y: auto; `; -const Table = styled.table`; - border-collapse;: collapse; +const Table = styled.table` + border-collapse: collapse; width: 100%; `; diff --git a/packages/website/ts/pages/governance/connect_form.tsx b/packages/website/ts/pages/governance/connect_form.tsx index 96fcef7f09..129224d07f 100644 --- a/packages/website/ts/pages/governance/connect_form.tsx +++ b/packages/website/ts/pages/governance/connect_form.tsx @@ -220,20 +220,14 @@ export class ConnectForm extends React.Component { } public async getZrxBalanceAsync(owner: string): Promise { utils.assert(!_.isUndefined(this._contractWrappers), 'ContractWrappers must be instantiated.'); - const injectedProvider = await this._getInjectedProviderIfExistsAsync(); - - if (!_.isUndefined(injectedProvider)) { - const contractAddresses = getContractAddressesForNetworkOrThrow(this.networkId); - const tokenAddress: string = contractAddresses.zrxToken; - try { - const amount = await this._contractWrappers.erc20Token.getBalanceAsync(tokenAddress, owner); - return amount; - } catch (error) { - return ZERO; - } + const contractAddresses = getContractAddressesForNetworkOrThrow(this.networkId); + const tokenAddress: string = contractAddresses.zrxToken; + try { + const amount = await this._contractWrappers.erc20Token.getBalanceAsync(tokenAddress, owner); + return amount; + } catch (error) { + return ZERO; } - - return ZERO; } private async _onConnectWalletClickAsync(): Promise { const shouldUseLedgerProvider = false; diff --git a/packages/website/ts/pages/governance/ledger_sign_note.tsx b/packages/website/ts/pages/governance/ledger_sign_note.tsx new file mode 100644 index 0000000000..3513cebf33 --- /dev/null +++ b/packages/website/ts/pages/governance/ledger_sign_note.tsx @@ -0,0 +1,48 @@ +import * as _ from 'lodash'; +import * as React from 'react'; +import styled from 'styled-components'; + +import { colors } from 'ts/style/colors'; + +interface WrapperProps { + isVisible: boolean; +} + +interface LedgerSignNoteProps extends WrapperProps { + text: string; +} + +export const LedgerSignNote: React.StatelessComponent = ({ text, isVisible }) => { + return ( + + {text} + + ); +}; + +LedgerSignNote.defaultProps = { + isVisible: false, +}; + +const Wrapper = styled.div` + background-color: #7a7a7a; + display: flex; + align-items: center; + padding: 28px 30px; + width: 100%; + position: absolute; + top: 0; + bottom: 0; + left: 0; + right: 0; + justify-content: center; + opacity: ${props => (props.isVisible ? 1 : 0)}; + visibility: ${props => (props.isVisible ? 'visible' : 'hidden')}; +`; + +const Text = styled.p` + color: ${colors.white}; + font-size: 1rem; + line-height: 1; + font-weight: 400; +`; diff --git a/packages/website/ts/pages/governance/modal_vote.tsx b/packages/website/ts/pages/governance/modal_vote.tsx index a86b95f2f0..0d0898d1ef 100644 --- a/packages/website/ts/pages/governance/modal_vote.tsx +++ b/packages/website/ts/pages/governance/modal_vote.tsx @@ -120,7 +120,7 @@ export class ModalVote extends React.Component { - Vote Recieved! + Vote Received! Your vote will help to decide the future of the protocol. You will be receiving a custom diff --git a/packages/website/ts/pages/governance/vote_bar.tsx b/packages/website/ts/pages/governance/vote_bar.tsx index 797edeb652..a9ec957809 100644 --- a/packages/website/ts/pages/governance/vote_bar.tsx +++ b/packages/website/ts/pages/governance/vote_bar.tsx @@ -1,3 +1,4 @@ +import { BigNumber } from '@0x/utils'; import * as _ from 'lodash'; import * as React from 'react'; import styled from 'styled-components'; @@ -5,7 +6,7 @@ import styled from 'styled-components'; interface VoteBarProps { label: string; color: string; - percentage: string; + percentage: BigNumber; marginBottom?: string; } @@ -14,16 +15,27 @@ interface VoteColumnProps { width: string; } -export const VoteBar: React.StatelessComponent = ({ percentage, color, label, marginBottom }) => { - const percentageLabel = `${percentage}%`; +const buildVotePercentageLabel = (percentage: BigNumber): string => { + let percentageLabel = `${percentage.toFixed(0)}%`; + // When voting is entirely dominated it can result in showing 100% and 0% + // In this case we replace with an indication that there are some votes for + // the minority + if (percentage.isGreaterThan(99) && percentage.isLessThan(100)) { + percentageLabel = `> 99%`; + } else if (percentage.isGreaterThan(0) && percentage.isLessThan(1)) { + percentageLabel = `< 1%`; + } + return percentageLabel; +}; +export const VoteBar: React.StatelessComponent = ({ percentage, color, label, marginBottom }) => { // TODO convert this to use a Container component return ( {label}
- - {percentageLabel} + + {buildVotePercentageLabel(percentage)}
); @@ -33,7 +45,7 @@ const VoteColumn = styled.div` background-color: ${props => props.color}; width: calc(${props => props.width}% - 45px); height: 13px; - margin-right: 10px; + margin-right: 15px; min-width: 10px; `; @@ -55,4 +67,5 @@ const VoteColumnLabel = styled.span` font-size: 1rem; line-height: 1; font-weight: 300; + min-width: 60px; `; diff --git a/packages/website/ts/pages/governance/vote_form.tsx b/packages/website/ts/pages/governance/vote_form.tsx index 4155f37789..32f1b1f0df 100644 --- a/packages/website/ts/pages/governance/vote_form.tsx +++ b/packages/website/ts/pages/governance/vote_form.tsx @@ -14,6 +14,7 @@ import styled from 'styled-components'; import { Button } from 'ts/components/button'; import { Input } from 'ts/components/modals/input'; import { Heading, Paragraph } from 'ts/components/text'; +import { LedgerSignNote } from 'ts/pages/governance/ledger_sign_note'; import { PreferenceSelecter } from 'ts/pages/governance/preference_selecter'; import { colors } from 'ts/style/colors'; import { InjectedProvider } from 'ts/types'; @@ -50,6 +51,7 @@ interface State { isWalletConnected: boolean; isSubmitting: boolean; isSuccessful: boolean; + isAwaitingLedgerSignature: boolean; isVoted: boolean; selectedAddress?: string; votePreference?: string; @@ -95,6 +97,7 @@ export class VoteForm extends React.Component { public networkId: number; public state: State = { isWalletConnected: false, + isAwaitingLedgerSignature: false, isSubmitting: false, isSuccessful: false, isVoted: false, @@ -116,7 +119,7 @@ export class VoteForm extends React.Component { super(props); } public render(): React.ReactNode { - const { votePreference, errors, isSuccessful } = this.state; + const { votePreference, errors, isSuccessful, isAwaitingLedgerSignature } = this.state; const { currentBalance, selectedAddress } = this.props; const bigNumberFormat = { decimalSeparator: '.', @@ -185,6 +188,10 @@ export class VoteForm extends React.Component { Back Submit + ); @@ -193,15 +200,19 @@ export class VoteForm extends React.Component { e.preventDefault(); const { zeip, votePreference, comment } = this.state; - const { currentBalance, selectedAddress } = this.props; + const { currentBalance, selectedAddress, isLedger } = this.props; const makerAddress = selectedAddress; + + if (isLedger) { + this.setState({ isAwaitingLedgerSignature: true }); + } + const domainType = [{ name: 'name', type: 'string' }]; const voteType = [ { name: 'preference', type: 'string' }, { name: 'zeip', type: 'uint256' }, { name: 'from', type: 'address' }, ]; - const domainData = { name: '0x Protocol Governance', }; @@ -224,8 +235,14 @@ export class VoteForm extends React.Component { const voteHashHex = `0x${voteHashBuffer.toString('hex')}`; try { const signedVote = await this._signVoteAsync(makerAddress, typedData); - // Store the signed Order - this.setState(prevState => ({ ...prevState, signedVote, voteHash: voteHashHex, isSuccessful: true })); + // Store the signed vote + this.setState(prevState => ({ + ...prevState, + signedVote, + voteHash: voteHashHex, + isSuccessful: true, + isAwaitingLedgerSignature: false, + })); const voteDomain = utils.isProduction() ? `https://${configs.DOMAIN_VOTE}` : 'http://localhost:3000'; const voteEndpoint = `${voteDomain}/v1/vote`; @@ -248,27 +265,27 @@ export class VoteForm extends React.Component { } else { const responseBody = await response.json(); const errorMessage = !_.isUndefined(responseBody.reason) ? responseBody.reason : 'Unknown Error'; - this.props.onError - ? this.props.onError(errorMessage) - : this.setState({ - errors: { - signError: errorMessage, - }, - isSuccessful: false, - }); + this._handleError(errorMessage); } } catch (err) { - const errorMessage = err.message; - this.props.onError - ? this.props.onError(errorMessage) - : this.setState({ - errors: { - signError: errorMessage, - }, - isSuccessful: false, - }); + this._handleError(err.message); } }; + private _handleError(errorMessage: string): void { + const { onError } = this.props; + onError + ? onError(errorMessage) + : this.setState({ + errors: { + signError: errorMessage, + }, + isSuccessful: false, + isAwaitingLedgerSignature: false, + }); + this.setState({ + isAwaitingLedgerSignature: false, + }); + } private async _signVoteAsync(signerAddress: string, typedData: any): Promise { const { provider: providerEngine } = this.props; let signatureHex; @@ -340,6 +357,8 @@ const InputRow = styled.div` `; const ButtonRow = styled(InputRow)` + position: relative; + @media (max-width: 768px) { display: flex; flex-direction: column; diff --git a/packages/website/ts/pages/governance/vote_stats.tsx b/packages/website/ts/pages/governance/vote_stats.tsx index c57a91e0f1..f2c8a38bee 100644 --- a/packages/website/ts/pages/governance/vote_stats.tsx +++ b/packages/website/ts/pages/governance/vote_stats.tsx @@ -11,6 +11,7 @@ import { constants } from 'ts/utils/constants'; interface VoteStatsProps { tally?: TallyInterface; } + export const VoteStats: React.StatelessComponent = ({ tally }) => { const bigNumberFormat = { decimalSeparator: '.', @@ -43,8 +44,8 @@ export const VoteStats: React.StatelessComponent = ({ tally }) = Results - - + + ({totalBalanceString} ZRX total vote) ); diff --git a/yarn.lock b/yarn.lock index 019ad3daf0..8a4384cb6e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -606,6 +606,14 @@ dependencies: npm-registry-client "7.0.9" +"@0xproject/types@^1.0.1-rc.3": + version "1.1.4" + resolved "https://registry.npmjs.org/@0xproject/types/-/types-1.1.4.tgz#3ffd65e670d6a21dab19ee0ffd5fad0056291b8e" + dependencies: + "@types/node" "*" + bignumber.js "~4.1.0" + ethereum-types "^1.0.11" + "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.0.0-beta.35": version "7.0.0" resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz#06e2ab19bdb535385559aabb5ba59729482800f8" @@ -1280,6 +1288,13 @@ call-me-maybe "^1.0.1" glob-to-regexp "^0.3.0" +"@radarrelay/types@^1.2.1": + version "1.2.1" + resolved "https://registry.npmjs.org/@radarrelay/types/-/types-1.2.1.tgz#d16edb43d0735a31c887b9e79ff6e53924ac8cc5" + dependencies: + "@0xproject/types" "^1.0.1-rc.3" + bignumber.js "^5.0.0" + "@reach/component-component@^0.1.1": version "0.1.1" resolved "https://registry.yarnpkg.com/@reach/component-component/-/component-component-0.1.1.tgz#62ea2ec290da32f5e3a9872fb51f9a3ae4370cc4" @@ -3439,6 +3454,10 @@ bignumber.js@7.2.1: version "7.2.1" resolved "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz#80c048759d826800807c4bfd521e50edbba57a5f" +bignumber.js@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/bignumber.js/-/bignumber.js-5.0.0.tgz#fbce63f09776b3000a83185badcde525daf34833" + "bignumber.js@git+https://github.com/debris/bignumber.js.git#94d7146671b9719e00a09c29b01a691bc85048c2": version "2.0.7" resolved "git+https://github.com/debris/bignumber.js.git#94d7146671b9719e00a09c29b01a691bc85048c2" @@ -6411,6 +6430,13 @@ ethereum-common@^0.0.18: version "0.0.18" resolved "https://registry.yarnpkg.com/ethereum-common/-/ethereum-common-0.0.18.tgz#2fdc3576f232903358976eb39da783213ff9523f" +ethereum-types@^1.0.11: + version "1.1.6" + resolved "https://registry.npmjs.org/ethereum-types/-/ethereum-types-1.1.6.tgz#14437dbf401de361e70dac6358e5f2915ad3c35d" + dependencies: + "@types/node" "*" + bignumber.js "~4.1.0" + ethereumjs-abi@0.6.5: version "0.6.5" resolved "https://registry.yarnpkg.com/ethereumjs-abi/-/ethereumjs-abi-0.6.5.tgz#5a637ef16ab43473fa72a29ad90871405b3f5241" @@ -7919,7 +7945,7 @@ got@^6.7.1: graceful-fs@4.1.15, graceful-fs@^3.0.0, graceful-fs@^4.0.0, graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9, graceful-fs@~1.2.0: version "4.1.15" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00" + resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00" "graceful-readlink@>= 1.0.0": version "1.0.1" @@ -13472,6 +13498,15 @@ react-dom@^16.3.2: object-assign "^4.1.1" prop-types "^15.6.0" +react-dom@^16.4.2: + version "16.8.1" + resolved "https://registry.npmjs.org/react-dom/-/react-dom-16.8.1.tgz#ec860f98853d09d39bafd3a6f1e12389d283dbb4" + dependencies: + loose-envify "^1.1.0" + object-assign "^4.1.1" + prop-types "^15.6.2" + scheduler "^0.13.1" + react-dom@^16.5.2: version "16.5.2" resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-16.5.2.tgz#b69ee47aa20bab5327b2b9d7c1fe2a30f2cfa9d7" @@ -13775,6 +13810,15 @@ react@^16.3.2: object-assign "^4.1.1" prop-types "^15.6.0" +react@^16.4.2: + version "16.8.1" + resolved "https://registry.npmjs.org/react/-/react-16.8.1.tgz#ae11831f6cb2a05d58603a976afc8a558e852c4a" + dependencies: + loose-envify "^1.1.0" + object-assign "^4.1.1" + prop-types "^15.6.2" + scheduler "^0.13.1" + react@^16.5.2: version "16.5.2" resolved "https://registry.yarnpkg.com/react/-/react-16.5.2.tgz#19f6b444ed139baa45609eee6dc3d318b3895d42" @@ -14646,6 +14690,13 @@ schedule@^0.5.0: dependencies: object-assign "^4.1.1" +scheduler@^0.13.1: + version "0.13.1" + resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.13.1.tgz#1a217df1bfaabaf4f1b92a9127d5d732d85a9591" + dependencies: + loose-envify "^1.1.0" + object-assign "^4.1.1" + schema-utils@^0.4.4: version "0.4.7" resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.7.tgz#ba74f597d2be2ea880131746ee17d0a093c68187"