diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index b2ad39a5f2..ecb53e5da3 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -49,6 +49,7 @@ "@0x/types": "^2.0.2", "@0x/utils": "^4.0.3", "@0x/web3-wrapper": "^4.0.2", + "@radarrelay/types": "^1.2.1", "@types/dockerode": "^2.5.9", "@types/p-limit": "^2.0.0", "async-parallel": "^1.2.3", diff --git a/packages/pipeline/src/data_sources/radar/index.ts b/packages/pipeline/src/data_sources/radar/index.ts index 7f7abb3334..bbc55eef99 100644 --- a/packages/pipeline/src/data_sources/radar/index.ts +++ b/packages/pipeline/src/data_sources/radar/index.ts @@ -27,7 +27,7 @@ export class RadarSource { */ public async getMarketOrderbookAsync(marketId: string): Promise { logUtils.log(`${marketId}: Retrieving orderbook.`); - const marketOrderbookUrl = `${ACTIVE_MARKETS_URL}/${marketId}/book`; + const marketOrderbookUrl = `${ACTIVE_MARKETS_URL}/${marketId}/book?perPage=${MAX_PER_PAGE}`; const resp = await fetchAsync(marketOrderbookUrl); return resp.json(); } diff --git a/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts new file mode 100644 index 0000000000..91d33d2cc9 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts @@ -0,0 +1,56 @@ +import { logUtils } from '@0x/utils'; +import { RadarMarket } from '@radarrelay/types'; +import * as R from 'ramda'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { RADAR_SOURCE, RadarSource } from '../data_sources/radar'; +import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseRadarOrders } from '../parsers/radar_orders'; +import { handleError } from '../utils'; + +// Number of orders to save at once. +const BATCH_SAVE_SIZE = 1000; + +// Number of markets to retrieve orderbooks for at once. +const MARKET_ORDERBOOK_REQUEST_BATCH_SIZE = 50; + +// Delay between market orderbook requests. +const MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY = 5000; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + const radarSource = new RadarSource(); + const markets = await radarSource.getActiveMarketsAsync(); + for (const marketsChunk of R.splitEvery(MARKET_ORDERBOOK_REQUEST_BATCH_SIZE, markets)) { + await Promise.all( + marketsChunk.map(async (market: RadarMarket) => getAndSaveMarketOrderbookAsync(radarSource, market)), + ); + await new Promise(resolve => setTimeout(resolve, MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY)); + } + process.exit(0); +})().catch(handleError); + +/** + * Retrieve orderbook from radar API for a given market. Parse orders and insert + * them into our database. + * @param radarSource Data source which can query radar API. + * @param market Object from radar API containing market data. + */ +async function getAndSaveMarketOrderbookAsync(radarSource: RadarSource, market: RadarMarket): Promise { + const orderBook = await radarSource.getMarketOrderbookAsync(market.id); + const observedTimestamp = Date.now(); + + logUtils.log(`${market.id}: Parsing orders.`); + const orders = parseRadarOrders(orderBook, market, observedTimestamp, RADAR_SOURCE); + + if (orders.length > 0) { + logUtils.log(`${market.id}: Saving ${orders.length} orders.`); + const TokenOrderRepository = connection.getRepository(TokenOrder); + await TokenOrderRepository.save(orders, { chunk: Math.ceil(orders.length / BATCH_SAVE_SIZE) }); + } else { + logUtils.log(`${market.id}: 0 orders to save.`); + } +} diff --git a/yarn.lock b/yarn.lock index fae178e3e4..444b5eeed7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -593,6 +593,14 @@ dependencies: npm-registry-client "7.0.9" +"@0xproject/types@^1.0.1-rc.3": + version "1.1.4" + resolved "https://registry.npmjs.org/@0xproject/types/-/types-1.1.4.tgz#3ffd65e670d6a21dab19ee0ffd5fad0056291b8e" + dependencies: + "@types/node" "*" + bignumber.js "~4.1.0" + ethereum-types "^1.0.11" + "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.0.0-beta.35": version "7.0.0" resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz#06e2ab19bdb535385559aabb5ba59729482800f8" @@ -1267,6 +1275,13 @@ call-me-maybe "^1.0.1" glob-to-regexp "^0.3.0" +"@radarrelay/types@^1.2.1": + version "1.2.1" + resolved "https://registry.npmjs.org/@radarrelay/types/-/types-1.2.1.tgz#d16edb43d0735a31c887b9e79ff6e53924ac8cc5" + dependencies: + "@0xproject/types" "^1.0.1-rc.3" + bignumber.js "^5.0.0" + "@reach/component-component@^0.1.1": version "0.1.1" resolved "https://registry.yarnpkg.com/@reach/component-component/-/component-component-0.1.1.tgz#62ea2ec290da32f5e3a9872fb51f9a3ae4370cc4" @@ -3432,6 +3447,10 @@ bignumber.js@7.2.1: version "7.2.1" resolved "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz#80c048759d826800807c4bfd521e50edbba57a5f" +bignumber.js@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/bignumber.js/-/bignumber.js-5.0.0.tgz#fbce63f09776b3000a83185badcde525daf34833" + "bignumber.js@git+https://github.com/debris/bignumber.js.git#94d7146671b9719e00a09c29b01a691bc85048c2": version "2.0.7" resolved "git+https://github.com/debris/bignumber.js.git#94d7146671b9719e00a09c29b01a691bc85048c2" @@ -6404,6 +6423,13 @@ ethereum-common@^0.0.18: version "0.0.18" resolved "https://registry.yarnpkg.com/ethereum-common/-/ethereum-common-0.0.18.tgz#2fdc3576f232903358976eb39da783213ff9523f" +ethereum-types@^1.0.11: + version "1.1.6" + resolved "https://registry.npmjs.org/ethereum-types/-/ethereum-types-1.1.6.tgz#14437dbf401de361e70dac6358e5f2915ad3c35d" + dependencies: + "@types/node" "*" + bignumber.js "~4.1.0" + ethereumjs-abi@0.6.5: version "0.6.5" resolved "https://registry.yarnpkg.com/ethereumjs-abi/-/ethereumjs-abi-0.6.5.tgz#5a637ef16ab43473fa72a29ad90871405b3f5241" @@ -7910,9 +7936,19 @@ got@^6.7.1: unzip-response "^2.0.1" url-parse-lax "^1.0.0" -graceful-fs@4.1.15, graceful-fs@^3.0.0, graceful-fs@^4.0.0, graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9, graceful-fs@~1.2.0: +graceful-fs@^3.0.0: + version "3.0.11" + resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-3.0.11.tgz#7613c778a1afea62f25c630a086d7f3acbbdd818" + dependencies: + natives "^1.1.0" + +graceful-fs@^4.0.0, graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9: version "4.1.15" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00" + resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00" + +graceful-fs@~1.2.0: + version "1.2.3" + resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-1.2.3.tgz#15a4806a57547cb2d2dbf27f42e89a8c3451b364" "graceful-readlink@>= 1.0.0": version "1.0.1" @@ -11343,6 +11379,10 @@ nanomatch@^1.2.9: snapdragon "^0.8.1" to-regex "^3.0.1" +natives@^1.1.0: + version "1.1.6" + resolved "https://registry.npmjs.org/natives/-/natives-1.1.6.tgz#a603b4a498ab77173612b9ea1acdec4d980f00bb" + natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" @@ -13463,8 +13503,7 @@ react-dom@^16.3.2: react-dom@^16.4.2: version "16.8.1" - resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-16.8.1.tgz#ec860f98853d09d39bafd3a6f1e12389d283dbb4" - integrity sha512-N74IZUrPt6UiDjXaO7UbDDFXeUXnVhZzeRLy/6iqqN1ipfjrhR60Bp5NuBK+rv3GMdqdIuwIl22u1SYwf330bg== + resolved "https://registry.npmjs.org/react-dom/-/react-dom-16.8.1.tgz#ec860f98853d09d39bafd3a6f1e12389d283dbb4" dependencies: loose-envify "^1.1.0" object-assign "^4.1.1" @@ -13528,8 +13567,8 @@ react-highlight@0xproject/react-highlight#react-peer-deps: dependencies: highlight.js "^9.11.0" highlightjs-solidity "^1.0.5" - react "^16.5.2" - react-dom "^16.5.2" + react "^16.4.2" + react-dom "^16.4.2" react-hot-loader@^4.3.3: version "4.3.4" @@ -13776,8 +13815,7 @@ react@^16.3.2: react@^16.4.2: version "16.8.1" - resolved "https://registry.yarnpkg.com/react/-/react-16.8.1.tgz#ae11831f6cb2a05d58603a976afc8a558e852c4a" - integrity sha512-wLw5CFGPdo7p/AgteFz7GblI2JPOos0+biSoxf1FPsGxWQZdN/pj6oToJs1crn61DL3Ln7mN86uZ4j74p31ELQ== + resolved "https://registry.npmjs.org/react/-/react-16.8.1.tgz#ae11831f6cb2a05d58603a976afc8a558e852c4a" dependencies: loose-envify "^1.1.0" object-assign "^4.1.1" @@ -14657,8 +14695,7 @@ schedule@^0.5.0: scheduler@^0.13.1: version "0.13.1" - resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.13.1.tgz#1a217df1bfaabaf4f1b92a9127d5d732d85a9591" - integrity sha512-VJKOkiKIN2/6NOoexuypwSrybx13MY7NSy9RNt8wPvZDMRT1CW6qlpF5jXRToXNHz3uWzbm2elNpZfXfGPqP9A== + resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.13.1.tgz#1a217df1bfaabaf4f1b92a9127d5d732d85a9591" dependencies: loose-envify "^1.1.0" object-assign "^4.1.1"