Add script for pulling radar orderbook
This commit is contained in:
fragosti
parent
f3a537d5c2
commit
4db9b8b0e3
@ -49,6 +49,7 @@
|
||||
"@0x/types": "^2.0.2",
|
||||
"@0x/utils": "^4.0.3",
|
||||
"@0x/web3-wrapper": "^4.0.2",
|
||||
"@radarrelay/types": "^1.2.1",
|
||||
"@types/dockerode": "^2.5.9",
|
||||
"@types/p-limit": "^2.0.0",
|
||||
"async-parallel": "^1.2.3",
|
||||
|
||||
@ -27,7 +27,7 @@ export class RadarSource {
|
||||
*/
|
||||
public async getMarketOrderbookAsync(marketId: string): Promise<RadarBook> {
|
||||
logUtils.log(`${marketId}: Retrieving orderbook.`);
|
||||
const marketOrderbookUrl = `${ACTIVE_MARKETS_URL}/${marketId}/book`;
|
||||
const marketOrderbookUrl = `${ACTIVE_MARKETS_URL}/${marketId}/book?perPage=${MAX_PER_PAGE}`;
|
||||
const resp = await fetchAsync(marketOrderbookUrl);
|
||||
return resp.json();
|
||||
}
|
||||
|
||||
@ -0,0 +1,56 @@
|
||||
import { logUtils } from '@0x/utils';
|
||||
import { RadarMarket } from '@radarrelay/types';
|
||||
import * as R from 'ramda';
|
||||
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
|
||||
|
||||
import { RADAR_SOURCE, RadarSource } from '../data_sources/radar';
|
||||
import { TokenOrderbookSnapshot as TokenOrder } from '../entities';
|
||||
import * as ormConfig from '../ormconfig';
|
||||
import { parseRadarOrders } from '../parsers/radar_orders';
|
||||
import { handleError } from '../utils';
|
||||
|
||||
// Number of orders to save at once.
|
||||
const BATCH_SAVE_SIZE = 1000;
|
||||
|
||||
// Number of markets to retrieve orderbooks for at once.
|
||||
const MARKET_ORDERBOOK_REQUEST_BATCH_SIZE = 50;
|
||||
|
||||
// Delay between market orderbook requests.
|
||||
const MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY = 5000;
|
||||
|
||||
let connection: Connection;
|
||||
|
||||
(async () => {
|
||||
connection = await createConnection(ormConfig as ConnectionOptions);
|
||||
const radarSource = new RadarSource();
|
||||
const markets = await radarSource.getActiveMarketsAsync();
|
||||
for (const marketsChunk of R.splitEvery(MARKET_ORDERBOOK_REQUEST_BATCH_SIZE, markets)) {
|
||||
await Promise.all(
|
||||
marketsChunk.map(async (market: RadarMarket) => getAndSaveMarketOrderbookAsync(radarSource, market)),
|
||||
);
|
||||
await new Promise<void>(resolve => setTimeout(resolve, MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY));
|
||||
}
|
||||
process.exit(0);
|
||||
})().catch(handleError);
|
||||
|
||||
/**
|
||||
* Retrieve orderbook from radar API for a given market. Parse orders and insert
|
||||
* them into our database.
|
||||
* @param radarSource Data source which can query radar API.
|
||||
* @param market Object from radar API containing market data.
|
||||
*/
|
||||
async function getAndSaveMarketOrderbookAsync(radarSource: RadarSource, market: RadarMarket): Promise<void> {
|
||||
const orderBook = await radarSource.getMarketOrderbookAsync(market.id);
|
||||
const observedTimestamp = Date.now();
|
||||
|
||||
logUtils.log(`${market.id}: Parsing orders.`);
|
||||
const orders = parseRadarOrders(orderBook, market, observedTimestamp, RADAR_SOURCE);
|
||||
|
||||
if (orders.length > 0) {
|
||||
logUtils.log(`${market.id}: Saving ${orders.length} orders.`);
|
||||
const TokenOrderRepository = connection.getRepository(TokenOrder);
|
||||
await TokenOrderRepository.save(orders, { chunk: Math.ceil(orders.length / BATCH_SAVE_SIZE) });
|
||||
} else {
|
||||
logUtils.log(`${market.id}: 0 orders to save.`);
|
||||
}
|
||||
}
|
||||
57
yarn.lock
57
yarn.lock
@ -593,6 +593,14 @@
|
||||
dependencies:
|
||||
npm-registry-client "7.0.9"
|
||||
|
||||
"@0xproject/types@^1.0.1-rc.3":
|
||||
version "1.1.4"
|
||||
resolved "https://registry.npmjs.org/@0xproject/types/-/types-1.1.4.tgz#3ffd65e670d6a21dab19ee0ffd5fad0056291b8e"
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
bignumber.js "~4.1.0"
|
||||
ethereum-types "^1.0.11"
|
||||
|
||||
"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.0.0-beta.35":
|
||||
version "7.0.0"
|
||||
resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz#06e2ab19bdb535385559aabb5ba59729482800f8"
|
||||
@ -1267,6 +1275,13 @@
|
||||
call-me-maybe "^1.0.1"
|
||||
glob-to-regexp "^0.3.0"
|
||||
|
||||
"@radarrelay/types@^1.2.1":
|
||||
version "1.2.1"
|
||||
resolved "https://registry.npmjs.org/@radarrelay/types/-/types-1.2.1.tgz#d16edb43d0735a31c887b9e79ff6e53924ac8cc5"
|
||||
dependencies:
|
||||
"@0xproject/types" "^1.0.1-rc.3"
|
||||
bignumber.js "^5.0.0"
|
||||
|
||||
"@reach/component-component@^0.1.1":
|
||||
version "0.1.1"
|
||||
resolved "https://registry.yarnpkg.com/@reach/component-component/-/component-component-0.1.1.tgz#62ea2ec290da32f5e3a9872fb51f9a3ae4370cc4"
|
||||
@ -3432,6 +3447,10 @@ bignumber.js@7.2.1:
|
||||
version "7.2.1"
|
||||
resolved "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz#80c048759d826800807c4bfd521e50edbba57a5f"
|
||||
|
||||
bignumber.js@^5.0.0:
|
||||
version "5.0.0"
|
||||
resolved "https://registry.npmjs.org/bignumber.js/-/bignumber.js-5.0.0.tgz#fbce63f09776b3000a83185badcde525daf34833"
|
||||
|
||||
"bignumber.js@git+https://github.com/debris/bignumber.js.git#94d7146671b9719e00a09c29b01a691bc85048c2":
|
||||
version "2.0.7"
|
||||
resolved "git+https://github.com/debris/bignumber.js.git#94d7146671b9719e00a09c29b01a691bc85048c2"
|
||||
@ -6404,6 +6423,13 @@ ethereum-common@^0.0.18:
|
||||
version "0.0.18"
|
||||
resolved "https://registry.yarnpkg.com/ethereum-common/-/ethereum-common-0.0.18.tgz#2fdc3576f232903358976eb39da783213ff9523f"
|
||||
|
||||
ethereum-types@^1.0.11:
|
||||
version "1.1.6"
|
||||
resolved "https://registry.npmjs.org/ethereum-types/-/ethereum-types-1.1.6.tgz#14437dbf401de361e70dac6358e5f2915ad3c35d"
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
bignumber.js "~4.1.0"
|
||||
|
||||
ethereumjs-abi@0.6.5:
|
||||
version "0.6.5"
|
||||
resolved "https://registry.yarnpkg.com/ethereumjs-abi/-/ethereumjs-abi-0.6.5.tgz#5a637ef16ab43473fa72a29ad90871405b3f5241"
|
||||
@ -7910,9 +7936,19 @@ got@^6.7.1:
|
||||
unzip-response "^2.0.1"
|
||||
url-parse-lax "^1.0.0"
|
||||
|
||||
graceful-fs@4.1.15, graceful-fs@^3.0.0, graceful-fs@^4.0.0, graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9, graceful-fs@~1.2.0:
|
||||
graceful-fs@^3.0.0:
|
||||
version "3.0.11"
|
||||
resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-3.0.11.tgz#7613c778a1afea62f25c630a086d7f3acbbdd818"
|
||||
dependencies:
|
||||
natives "^1.1.0"
|
||||
|
||||
graceful-fs@^4.0.0, graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9:
|
||||
version "4.1.15"
|
||||
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00"
|
||||
resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00"
|
||||
|
||||
graceful-fs@~1.2.0:
|
||||
version "1.2.3"
|
||||
resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-1.2.3.tgz#15a4806a57547cb2d2dbf27f42e89a8c3451b364"
|
||||
|
||||
"graceful-readlink@>= 1.0.0":
|
||||
version "1.0.1"
|
||||
@ -11343,6 +11379,10 @@ nanomatch@^1.2.9:
|
||||
snapdragon "^0.8.1"
|
||||
to-regex "^3.0.1"
|
||||
|
||||
natives@^1.1.0:
|
||||
version "1.1.6"
|
||||
resolved "https://registry.npmjs.org/natives/-/natives-1.1.6.tgz#a603b4a498ab77173612b9ea1acdec4d980f00bb"
|
||||
|
||||
natural-compare@^1.4.0:
|
||||
version "1.4.0"
|
||||
resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
|
||||
@ -13463,8 +13503,7 @@ react-dom@^16.3.2:
|
||||
|
||||
react-dom@^16.4.2:
|
||||
version "16.8.1"
|
||||
resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-16.8.1.tgz#ec860f98853d09d39bafd3a6f1e12389d283dbb4"
|
||||
integrity sha512-N74IZUrPt6UiDjXaO7UbDDFXeUXnVhZzeRLy/6iqqN1ipfjrhR60Bp5NuBK+rv3GMdqdIuwIl22u1SYwf330bg==
|
||||
resolved "https://registry.npmjs.org/react-dom/-/react-dom-16.8.1.tgz#ec860f98853d09d39bafd3a6f1e12389d283dbb4"
|
||||
dependencies:
|
||||
loose-envify "^1.1.0"
|
||||
object-assign "^4.1.1"
|
||||
@ -13528,8 +13567,8 @@ react-highlight@0xproject/react-highlight#react-peer-deps:
|
||||
dependencies:
|
||||
highlight.js "^9.11.0"
|
||||
highlightjs-solidity "^1.0.5"
|
||||
react "^16.5.2"
|
||||
react-dom "^16.5.2"
|
||||
react "^16.4.2"
|
||||
react-dom "^16.4.2"
|
||||
|
||||
react-hot-loader@^4.3.3:
|
||||
version "4.3.4"
|
||||
@ -13776,8 +13815,7 @@ react@^16.3.2:
|
||||
|
||||
react@^16.4.2:
|
||||
version "16.8.1"
|
||||
resolved "https://registry.yarnpkg.com/react/-/react-16.8.1.tgz#ae11831f6cb2a05d58603a976afc8a558e852c4a"
|
||||
integrity sha512-wLw5CFGPdo7p/AgteFz7GblI2JPOos0+biSoxf1FPsGxWQZdN/pj6oToJs1crn61DL3Ln7mN86uZ4j74p31ELQ==
|
||||
resolved "https://registry.npmjs.org/react/-/react-16.8.1.tgz#ae11831f6cb2a05d58603a976afc8a558e852c4a"
|
||||
dependencies:
|
||||
loose-envify "^1.1.0"
|
||||
object-assign "^4.1.1"
|
||||
@ -14657,8 +14695,7 @@ schedule@^0.5.0:
|
||||
|
||||
scheduler@^0.13.1:
|
||||
version "0.13.1"
|
||||
resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.13.1.tgz#1a217df1bfaabaf4f1b92a9127d5d732d85a9591"
|
||||
integrity sha512-VJKOkiKIN2/6NOoexuypwSrybx13MY7NSy9RNt8wPvZDMRT1CW6qlpF5jXRToXNHz3uWzbm2elNpZfXfGPqP9A==
|
||||
resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.13.1.tgz#1a217df1bfaabaf4f1b92a9127d5d732d85a9591"
|
||||
dependencies:
|
||||
loose-envify "^1.1.0"
|
||||
object-assign "^4.1.1"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user