crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Factor out redundant source param in parse order functions

Browse Source
This commit is contained in:
fragosti
2019-02-20 13:45:14 -08:00
parent ef5195db90
commit 4952cd5afb
16 changed files with 37 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
packages/pipeline/src/parsers/ddex_orders/index.ts
View File

@@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils';
import { aggregateOrders } from '../utils';
import { DdexMarket, DdexOrderbook } from '../../data_sources/ddex';
import { DDEX_SOURCE, DdexMarket, DdexOrderbook } from '../../data_sources/ddex';
import { TokenOrderbookSnapshot as TokenOrder } from '../../entities';
import { OrderType } from '../../types';
@@ -13,22 +13,16 @@ import { OrderType } from '../../types';
* @param ddexOrderbook A raw orderbook that we pull from the Ddex API.
* @param ddexMarket An object containing market data also directly from the API.
* @param observedTimestamp Time at which the orders for the market were pulled.
* @param source The exchange where these orders are placed. In this case 'ddex'.
*/
export function parseDdexOrders(
ddexOrderbook: DdexOrderbook,
ddexMarket: DdexMarket,
observedTimestamp: number,
source: string,
): TokenOrder[] {
const aggregatedBids = aggregateOrders(ddexOrderbook.bids);
const aggregatedAsks = aggregateOrders(ddexOrderbook.asks);
const parsedBids = aggregatedBids.map(order =>
parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Bid, source, order),
);
const parsedAsks = aggregatedAsks.map(order =>
parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Ask, source, order),
);
const parsedBids = aggregatedBids.map(order => parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Bid, order));
const parsedAsks = aggregatedAsks.map(order => parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Ask, order));
return parsedBids.concat(parsedAsks);
}
@@ -46,14 +40,13 @@ export function parseDdexOrder(
ddexMarket: DdexMarket,
observedTimestamp: number,
orderType: OrderType,
source: string,
ddexOrder: [string, BigNumber],
): TokenOrder {
const tokenOrder = new TokenOrder();
const price = new BigNumber(ddexOrder[0]);
const amount = ddexOrder[1];
tokenOrder.source = source;
tokenOrder.source = DDEX_SOURCE;
tokenOrder.observedTimestamp = observedTimestamp;
tokenOrder.orderType = orderType;
tokenOrder.price = price;

17
packages/pipeline/src/parsers/idex_orders/index.ts
View File

@@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils';
import { aggregateOrders } from '../utils';
import { IdexOrderbook, IdexOrderParam } from '../../data_sources/idex';
import { IDEX_SOURCE, IdexOrderbook, IdexOrderParam } from '../../data_sources/idex';
import { TokenOrderbookSnapshot as TokenOrder } from '../../entities';
import { OrderType } from '../../types';
@@ -13,17 +13,14 @@ import { OrderType } from '../../types';
* 3) Parses them into entities which are then saved into the database.
* @param idexOrderbook raw orderbook that we pull from the Idex API.
* @param observedTimestamp Time at which the orders for the market were pulled.
* @param source The exchange where these orders are placed. In this case 'idex'.
*/
export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: number, source: string): TokenOrder[] {
export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: number): TokenOrder[] {
const aggregatedBids = aggregateOrders(idexOrderbook.bids);
// Any of the bid orders' params will work
const idexBidOrder = idexOrderbook.bids[0];
const parsedBids =
aggregatedBids.length > 0
? aggregatedBids.map(order =>
parseIdexOrder(idexBidOrder.params, observedTimestamp, OrderType.Bid, source, order),
)
? aggregatedBids.map(order => parseIdexOrder(idexBidOrder.params, observedTimestamp, OrderType.Bid, order))
: [];
const aggregatedAsks = aggregateOrders(idexOrderbook.asks);
@@ -31,9 +28,7 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp:
const idexAskOrder = idexOrderbook.asks[0];
const parsedAsks =
aggregatedAsks.length > 0
? aggregatedAsks.map(order =>
parseIdexOrder(idexAskOrder.params, observedTimestamp, OrderType.Ask, source, order),
)
? aggregatedAsks.map(order => parseIdexOrder(idexAskOrder.params, observedTimestamp, OrderType.Ask, order))
: [];
return parsedBids.concat(parsedAsks);
}
@@ -45,21 +40,19 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp:
* trades have been placed.
* @param observedTimestamp The time when the API response returned back to us.
* @param orderType 'bid' or 'ask' enum.
* @param source Exchange where these orders were placed.
* @param idexOrder A <price, amount> tuple which we will convert to volume-basis.
*/
export function parseIdexOrder(
idexOrderParam: IdexOrderParam,
observedTimestamp: number,
orderType: OrderType,
source: string,
idexOrder: [string, BigNumber],
): TokenOrder {
const tokenOrder = new TokenOrder();
const price = new BigNumber(idexOrder[0]);
const amount = idexOrder[1];
tokenOrder.source = source;
tokenOrder.source = IDEX_SOURCE;
tokenOrder.observedTimestamp = observedTimestamp;
tokenOrder.orderType = orderType;
tokenOrder.price = price;

11
packages/pipeline/src/parsers/oasis_orders/index.ts
View File

@@ -3,7 +3,7 @@ import * as R from 'ramda';
import { aggregateOrders } from '../utils';
import { OasisMarket, OasisOrder } from '../../data_sources/oasis';
import { OASIS_SOURCE, OasisMarket, OasisOrder } from '../../data_sources/oasis';
import { TokenOrderbookSnapshot as TokenOrder } from '../../entities';
import { OrderType } from '../../types';
@@ -15,21 +15,19 @@ import { OrderType } from '../../types';
* @param oasisOrderbook A raw orderbook that we pull from the Oasis API.
* @param oasisMarket An object containing market data also directly from the API.
* @param observedTimestamp Time at which the orders for the market were pulled.
* @param source The exchange where these orders are placed. In this case 'oasis'.
*/
export function parseOasisOrders(
oasisOrderbook: OasisOrder[],
oasisMarket: OasisMarket,
observedTimestamp: number,
source: string,
): TokenOrder[] {
const aggregatedBids = aggregateOrders(R.filter(R.propEq('act', OrderType.Bid), oasisOrderbook));
const aggregatedAsks = aggregateOrders(R.filter(R.propEq('act', OrderType.Ask), oasisOrderbook));
const parsedBids = aggregatedBids.map(order =>
parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Bid, source, order),
parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Bid, order),
);
const parsedAsks = aggregatedAsks.map(order =>
parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Ask, source, order),
parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Ask, order),
);
return parsedBids.concat(parsedAsks);
}
@@ -48,14 +46,13 @@ export function parseOasisOrder(
oasisMarket: OasisMarket,
observedTimestamp: number,
orderType: OrderType,
source: string,
oasisOrder: [string, BigNumber],
): TokenOrder {
const tokenOrder = new TokenOrder();
const price = new BigNumber(oasisOrder[0]);
const amount = oasisOrder[1];
tokenOrder.source = source;
tokenOrder.source = OASIS_SOURCE;
tokenOrder.observedTimestamp = observedTimestamp;
tokenOrder.orderType = orderType;
tokenOrder.price = price;

12
packages/pipeline/src/parsers/paradex_orders/index.ts
View File

@@ -1,6 +1,6 @@
import { BigNumber } from '@0x/utils';
import { ParadexMarket, ParadexOrder, ParadexOrderbookResponse } from '../../data_sources/paradex';
import { PARADEX_SOURCE, ParadexMarket, ParadexOrder, ParadexOrderbookResponse } from '../../data_sources/paradex';
import { TokenOrderbookSnapshot as TokenOrder } from '../../entities';
import { OrderType } from '../../types';
@@ -12,19 +12,17 @@ import { OrderType } from '../../types';
* @param paradexOrderbookResponse An orderbook response from the Paradex API.
* @param paradexMarket An object containing market data also directly from the API.
* @param observedTimestamp Time at which the orders for the market were pulled.
* @param source The exchange where these orders are placed. In this case 'paradex'.
*/
export function parseParadexOrders(
paradexOrderbookResponse: ParadexOrderbookResponse,
paradexMarket: ParadexMarket,
observedTimestamp: number,
source: string,
): TokenOrder[] {
const parsedBids = paradexOrderbookResponse.bids.map(order =>
parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Bid, source, order),
parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Bid, order),
);
const parsedAsks = paradexOrderbookResponse.asks.map(order =>
parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Ask, source, order),
parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Ask, order),
);
return parsedBids.concat(parsedAsks);
}
@@ -36,21 +34,19 @@ export function parseParadexOrders(
* orders have been placed.
* @param observedTimestamp The time when the API response returned back to us.
* @param orderType 'bid' or 'ask' enum.
* @param source Exchange where these orders were placed.
* @param paradexOrder A ParadexOrder object; basically price, amount tuple.
*/
export function parseParadexOrder(
paradexMarket: ParadexMarket,
observedTimestamp: number,
orderType: OrderType,
source: string,
paradexOrder: ParadexOrder,
): TokenOrder {
const tokenOrder = new TokenOrder();
const price = new BigNumber(paradexOrder.price);
const amount = new BigNumber(paradexOrder.amount);
tokenOrder.source = source;
tokenOrder.source = PARADEX_SOURCE;
tokenOrder.observedTimestamp = observedTimestamp;
tokenOrder.orderType = orderType;
tokenOrder.price = price;

11
packages/pipeline/src/parsers/radar_orders/index.ts
View File

@@ -5,6 +5,7 @@ import * as R from 'ramda';
import { aggregateOrders, GenericRawOrder } from '../utils';
import { RADAR_SOURCE } from '../../data_sources/radar';
import { TokenOrderbookSnapshot as TokenOrder } from '../../entities';
import { OrderType } from '../../types';
@@ -21,21 +22,19 @@ export interface AggregateOrdersByMaker {
* @param radarOrderbook A raw orderbook that we pull from the radar API.
* @param radarMarket An object containing market data also directly from the API.
* @param observedTimestamp Time at which the orders for the market were pulled.
* @param source The exchange where these orders are placed. In this case 'radar'.
*/
export function parseRadarOrders(
radarOrderbook: RadarBook,
radarMarket: RadarMarket,
observedTimestamp: number,
source: string,
): TokenOrder[] {
const aggregatedBids = _aggregateOrdersByMaker(radarMarket, radarOrderbook.bids);
const aggregatedAsks = _aggregateOrdersByMaker(radarMarket, radarOrderbook.asks);
const parsedBids = aggregatedBids.map(order =>
parseRadarOrder(radarMarket, observedTimestamp, OrderType.Bid, source, order),
parseRadarOrder(radarMarket, observedTimestamp, OrderType.Bid, order),
);
const parsedAsks = aggregatedAsks.map(order =>
parseRadarOrder(radarMarket, observedTimestamp, OrderType.Ask, source, order),
parseRadarOrder(radarMarket, observedTimestamp, OrderType.Ask, order),
);
return parsedBids.concat(parsedAsks);
}
@@ -47,14 +46,12 @@ export function parseRadarOrders(
* trades have been placed.
* @param observedTimestamp The time when the API response returned back to us.
* @param orderType 'bid' or 'ask' enum.
* @param source Exchange where these orders were placed.
* @param aggregateOrder An AggregateOrdersByMaker instance which we will convert to volume-basis.
*/
export function parseRadarOrder(
radarMarket: RadarMarket,
observedTimestamp: number,
orderType: OrderType,
source: string,
aggregateOrder: AggregateOrdersByMaker,
): TokenOrder {
const tokenOrder = new TokenOrder();
@@ -62,7 +59,7 @@ export function parseRadarOrder(
const amount = aggregateOrder.amount;
const splitId = radarMarket.id.split('-');
tokenOrder.source = source;
tokenOrder.source = RADAR_SOURCE;
tokenOrder.observedTimestamp = observedTimestamp;
tokenOrder.orderType = orderType;
tokenOrder.price = price;

4
packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts
View File

@@ -2,7 +2,7 @@ import { logUtils } from '@0x/utils';
import * as R from 'ramda';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import { DDEX_SOURCE, DdexMarket, DdexSource } from '../data_sources/ddex';
import { DdexMarket, DdexSource } from '../data_sources/ddex';
import { TokenOrderbookSnapshot as TokenOrder } from '../entities';
import * as ormConfig from '../ormconfig';
import { parseDdexOrders } from '../parsers/ddex_orders';
@@ -43,7 +43,7 @@ async function getAndSaveMarketOrderbookAsync(ddexSource: DdexSource, market: Dd
const observedTimestamp = Date.now();
logUtils.log(`${market.id}: Parsing orders.`);
const orders = parseDdexOrders(orderBook, market, observedTimestamp, DDEX_SOURCE);
const orders = parseDdexOrders(orderBook, market, observedTimestamp);
if (orders.length > 0) {
logUtils.log(`${market.id}: Saving ${orders.length} orders.`);

4
packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts
View File

@@ -2,7 +2,7 @@ import { logUtils } from '@0x/utils';
import * as R from 'ramda';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import { IDEX_SOURCE, IdexSource } from '../data_sources/idex';
import { IdexSource } from '../data_sources/idex';
import { TokenOrderbookSnapshot as TokenOrder } from '../entities';
import * as ormConfig from '../ormconfig';
import { parseIdexOrders } from '../parsers/idex_orders';
@@ -51,7 +51,7 @@ async function getAndSaveMarketOrderbookAsync(idexSource: IdexSource, marketId:
}
logUtils.log(`${marketId}: Parsing orders.`);
const orders = parseIdexOrders(orderBook, observedTimestamp, IDEX_SOURCE);
const orders = parseIdexOrders(orderBook, observedTimestamp);
if (orders.length > 0) {
logUtils.log(`${marketId}: Saving ${orders.length} orders.`);

4
packages/pipeline/src/scripts/pull_oasis_orderbook_snapshots.ts
View File

@@ -2,7 +2,7 @@ import { logUtils } from '@0x/utils';
import * as R from 'ramda';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import { OASIS_SOURCE, OasisMarket, OasisSource } from '../data_sources/oasis';
import { OasisMarket, OasisSource } from '../data_sources/oasis';
import { TokenOrderbookSnapshot as TokenOrder } from '../entities';
import * as ormConfig from '../ormconfig';
import { parseOasisOrders } from '../parsers/oasis_orders';
@@ -46,7 +46,7 @@ async function getAndSaveMarketOrderbookAsync(oasisSource: OasisSource, market:
const observedTimestamp = Date.now();
logUtils.log(`${market.id}: Parsing orders.`);
const orders = parseOasisOrders(orderBook, market, observedTimestamp, OASIS_SOURCE);
const orders = parseOasisOrders(orderBook, market, observedTimestamp);
if (orders.length > 0) {
logUtils.log(`${market.id}: Saving ${orders.length} orders.`);

3
packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts
View File

@@ -2,7 +2,6 @@ import { logUtils } from '@0x/utils';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import {
PARADEX_SOURCE,
ParadexActiveMarketsResponse,
ParadexMarket,
ParadexSource,
@@ -75,7 +74,7 @@ async function getAndSaveMarketOrderbookAsync(paradexSource: ParadexSource, mark
const observedTimestamp = Date.now();
logUtils.log(`${market.symbol}: Parsing orders.`);
const orders = parseParadexOrders(paradexOrderbookResponse, market, observedTimestamp, PARADEX_SOURCE);
const orders = parseParadexOrders(paradexOrderbookResponse, market, observedTimestamp);
if (orders.length > 0) {
logUtils.log(`${market.symbol}: Saving ${orders.length} orders.`);

4
packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts
View File

@@ -3,7 +3,7 @@ import { RadarMarket } from '@radarrelay/types';
import * as R from 'ramda';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import { RADAR_SOURCE, RadarSource } from '../data_sources/radar';
import { RadarSource } from '../data_sources/radar';
import { TokenOrderbookSnapshot as TokenOrder } from '../entities';
import * as ormConfig from '../ormconfig';
import { parseRadarOrders } from '../parsers/radar_orders';
@@ -44,7 +44,7 @@ async function getAndSaveMarketOrderbookAsync(radarSource: RadarSource, market:
const observedTimestamp = Date.now();
logUtils.log(`${market.id}: Parsing orders.`);
const orders = parseRadarOrders(orderBook, market, observedTimestamp, RADAR_SOURCE);
const orders = parseRadarOrders(orderBook, market, observedTimestamp);
if (orders.length > 0) {
logUtils.log(`${market.id}: Saving ${orders.length} orders.`);

1
packages/pipeline/test/data_sources/radar/index_test.ts
View File

@@ -2,7 +2,6 @@ import { BigNumber } from '@0x/utils';
import { RadarOrderState, RadarOrderType } from '@radarrelay/types';
import * as chai from 'chai';
import 'mocha';
import * as R from 'ramda';
import { RadarSource } from '../../../src/data_sources/radar';
import { chaiSetup } from '../../utils/chai_setup';

3
packages/pipeline/test/parsers/ddex_orders/index_test.ts
View File

@@ -31,7 +31,6 @@ describe('ddex_orders', () => {
};
const observedTimestamp: number = Date.now();
const orderType: OrderType = OrderType.Bid;
const source: string = 'ddex';
const expected = new TokenOrder();
expected.source = 'ddex';
@@ -45,7 +44,7 @@ describe('ddex_orders', () => {
expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81';
expected.baseVolume = new BigNumber(10);
expected.makerAddress = 'unknown';
const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, source, ddexOrder);
const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, ddexOrder);
expect(actual).deep.equal(expected);
});
});

6
packages/pipeline/test/parsers/idex_orders/index_test.ts
View File

@@ -32,7 +32,6 @@ describe('idex_orders', () => {
};
const observedTimestamp: number = Date.now();
const orderType: OrderType = OrderType.Bid;
const source: string = 'idex';
const expected = new TokenOrder();
expected.source = 'idex';
@@ -46,7 +45,7 @@ describe('idex_orders', () => {
expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81';
expected.quoteVolume = new BigNumber(5);
expected.makerAddress = 'unknown';
const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, source, idexOrder);
const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, idexOrder);
expect(actual).deep.equal(expected);
});
it('correctly converts ask type idexOrder to TokenOrder entity', () => {
@@ -66,7 +65,6 @@ describe('idex_orders', () => {
};
const observedTimestamp: number = Date.now();
const orderType: OrderType = OrderType.Ask;
const source: string = 'idex';
const expected = new TokenOrder();
expected.source = 'idex';
@@ -80,7 +78,7 @@ describe('idex_orders', () => {
expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81';
expected.quoteVolume = new BigNumber(5);
expected.makerAddress = 'unknown';
const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, source, idexOrder);
const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, idexOrder);
expect(actual).deep.equal(expected);
});
});

3
packages/pipeline/test/parsers/oasis_orders/index_test.ts
View File

@@ -28,7 +28,6 @@ describe('oasis_orders', () => {
};
const observedTimestamp: number = Date.now();
const orderType: OrderType = OrderType.Bid;
const source: string = 'oasis';
const expected = new TokenOrder();
expected.source = 'oasis';
@@ -42,7 +41,7 @@ describe('oasis_orders', () => {
expected.quoteAssetAddress = null;
expected.quoteVolume = new BigNumber(5);
expected.makerAddress = 'unknown';
const actual = parseOasisOrder(oasisMarket, observedTimestamp, orderType, source, oasisOrder);
const actual = parseOasisOrder(oasisMarket, observedTimestamp, orderType, oasisOrder);
expect(actual).deep.equal(expected);
});
});

3
packages/pipeline/test/parsers/paradex_orders/index_test.ts
View File

@@ -33,7 +33,6 @@ describe('paradex_orders', () => {
};
const observedTimestamp: number = Date.now();
const orderType: OrderType = OrderType.Bid;
const source: string = 'paradex';
const expected = new TokenOrder();
expected.source = 'paradex';
@@ -47,7 +46,7 @@ describe('paradex_orders', () => {
expected.quoteAssetAddress = '0x0000000000000000000000000000000000000000';
expected.quoteVolume = new BigNumber(412 * 0.1245);
expected.makerAddress = 'unknown';
const actual = parseParadexOrder(paradexMarket, observedTimestamp, orderType, source, paradexOrder);
const actual = parseParadexOrder(paradexMarket, observedTimestamp, orderType, paradexOrder);
expect(actual).deep.equal(expected);
});
});

5
packages/pipeline/test/parsers/radar_orders/index_test.ts
View File

@@ -1,5 +1,5 @@
import { BigNumber } from '@0x/utils';
import { RadarBook, RadarMarket, RadarSignedOrder } from '@radarrelay/types';
import { RadarMarket } from '@radarrelay/types';
import * as chai from 'chai';
import 'mocha';
@@ -35,7 +35,6 @@ describe('radar_orders', () => {
} as any) as RadarMarket;
const observedTimestamp: number = Date.now();
const orderType: OrderType = OrderType.Bid;
const source: string = 'radar';
const expected = new TokenOrder();
expected.source = 'radar';
@@ -49,7 +48,7 @@ describe('radar_orders', () => {
expected.baseAssetAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
expected.baseVolume = new BigNumber(10000000000);
expected.makerAddress = '0x6eC92694ea172ebC430C30fa31De87620967A082';
const actual = parseRadarOrder(radarMarket, observedTimestamp, orderType, source, radarOrder);
const actual = parseRadarOrder(radarMarket, observedTimestamp, orderType, radarOrder);
expect(actual).deep.equal(expected);
});
});