Integrate one-time dump and API for nonfungible.com (#1603)

* Add script for pulling NFT trade data from nonfungible.com * corrections for current state of API * change data model to match data source * change primary key * pull data from initial dump first, then API * pull all supported NFT's, not just cryptokitties * disable problematic data sources * rename function to satisfy linter * Rename table to nonfungible_dot_com_trades * rename parser module to nonfungible_dot_com from non_fungible_dot_com, for consistency * correct mistaken reference to Bloxy * rename NonfungibleDotComTrade to ...TradeResponse * `NftTrade` -> `NonfungibleDotComTrade` * rename files to match prior object renaming * use fetchAsync instead of axios * improve fetchAsync error message: include URL * avoid non-null contraints in API trades too, not just for trades from the one-time dump * disable mythereum publisher
2019-02-19 19:07:42 -08:00
parent e643c13292
commit 1232a9a03d
10 changed files with 522 additions and 1 deletions
--- a/packages/pipeline/src/scripts/pull_nonfungible_dot_com_trades.ts
+++ b/packages/pipeline/src/scripts/pull_nonfungible_dot_com_trades.ts
@@ -0,0 +1,43 @@
+// tslint:disable:no-console
+import 'reflect-metadata';
+import { Connection, ConnectionOptions, createConnection } from 'typeorm';
+
+import { getTradesAsync, knownPublishers } from '../data_sources/nonfungible_dot_com';
+import { NonfungibleDotComTrade } from '../entities';
+import * as ormConfig from '../ormconfig';
+import { parseNonFungibleDotComTrades } from '../parsers/nonfungible_dot_com';
+import { handleError } from '../utils';
+
+// Number of trades to save at once.
+const BATCH_SAVE_SIZE = 1000;
+
+let connection: Connection;
+
+(async () => {
+    connection = await createConnection(ormConfig as ConnectionOptions);
+    await getAndSaveTradesAsync();
+    process.exit(0);
+})().catch(handleError);
+
+async function getAndSaveTradesAsync(): Promise<void> {
+    const tradesRepository = connection.getRepository(NonfungibleDotComTrade);
+
+    for (const publisher of knownPublishers) {
+        console.log(`Getting latest trades for NFT ${publisher}...`);
+        const tradeWithHighestBlockNumber = await tradesRepository
+            .createQueryBuilder('nonfungible_dot_com_trades')
+            .where('nonfungible_dot_com_trades.publisher = :publisher', { publisher })
+            .orderBy({ 'nonfungible_dot_com_trades.block_number': 'DESC' })
+            .getOne();
+        const highestExistingBlockNumber =
+            tradeWithHighestBlockNumber === undefined ? 0 : tradeWithHighestBlockNumber.blockNumber;
+        console.log(`Highest block number in existing trades: ${highestExistingBlockNumber}`);
+        const rawTrades = await getTradesAsync(publisher, highestExistingBlockNumber);
+        console.log(`Parsing ${rawTrades.length} trades...`);
+        const trades = parseNonFungibleDotComTrades(rawTrades, publisher);
+        console.log(`Saving ${rawTrades.length} trades...`);
+        await tradesRepository.save(trades, { chunk: Math.ceil(trades.length / BATCH_SAVE_SIZE) });
+    }
+    const newTotalTrades = await tradesRepository.count();
+    console.log(`Done saving trades. There are now ${newTotalTrades} total NFT trades.`);
+}