crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split index.ts into multiple scripts in scripts/ and detect last known block when pulling events

Browse Source
This commit is contained in:
Alex Browne
2018-11-07 16:48:25 -08:00
parent 9cb89725c9
commit 954c3b9272
4 changed files with 95 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
packages/pipeline/src/index.ts
View File

@@ -1,88 +0,0 @@
import { web3Factory } from '@0x/dev-utils';
import { Web3ProviderEngine } from '@0x/subproviders';
import 'reflect-metadata';
import { Connection, createConnection } from 'typeorm';
import { ExchangeEventsSource } from './data_sources/contract-wrappers/exchange_events';
import { RelayerRegistrySource } from './data_sources/relayer-registry';
import { Web3Source } from './data_sources/web3';
import { Block } from './entities/Block';
import { ExchangeFillEvent } from './entities/ExchangeFillEvent';
import { Relayer } from './entities/Relayer';
import { Transaction } from './entities/Transaction';
import { deployConfig } from './ormconfig';
import { parseExchangeEvents } from './parsers/events';
import { parseRelayers } from './parsers/relayer_registry';
import { parseBlock, parseTransaction } from './parsers/web3';
const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
// NOTE(albrow): We need to manually update this URL for now. Fix this when we
// have the relayer-registry behind semantic versioning.
const RELAYER_REGISTRY_URL =
'https://raw.githubusercontent.com/0xProject/0x-relayer-registry/4701c85677d161ea729a466aebbc1826c6aa2c0b/relayers.json';
let connection: Connection;
(async () => {
connection = await createConnection(deployConfig);
const provider = web3Factory.getRpcProvider({
rpcUrl: 'https://mainnet.infura.io',
});
await getExchangeEventsAsync(provider);
await getBlockAsync(provider);
await getTransactionAsync(provider);
await getRelayers();
process.exit(0);
})();
async function getExchangeEventsAsync(provider: Web3ProviderEngine): Promise<void> {
console.log('Getting event logs...');
const eventsRepository = connection.getRepository(ExchangeFillEvent);
const exchangeEvents = new ExchangeEventsSource(provider, 1);
const eventLogs = await exchangeEvents.getFillEventsAsync(EXCHANGE_START_BLOCK, EXCHANGE_START_BLOCK + 100000);
console.log('Parsing events...');
const events = parseExchangeEvents(eventLogs);
console.log(`Retrieved and parsed ${events.length} total events.`);
console.log('Saving events...');
for (const event of events) {
await eventsRepository.save(event);
}
console.log('Saved events.');
}
async function getBlockAsync(provider: Web3ProviderEngine): Promise<void> {
console.log('Getting block info...');
const blocksRepository = connection.getRepository(Block);
const web3Source = new Web3Source(provider);
const rawBlock = await web3Source.getBlockInfoAsync(EXCHANGE_START_BLOCK);
const block = parseBlock(rawBlock);
console.log('Saving block info...');
await blocksRepository.save(block);
console.log('Done saving block.');
}
async function getTransactionAsync(provider: Web3ProviderEngine): Promise<void> {
console.log('Getting tx info...');
const txsRepository = connection.getRepository(Transaction);
const web3Source = new Web3Source(provider);
const rawTx = await web3Source.getTransactionInfoAsync(
'0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe',
);
const tx = parseTransaction(rawTx);
console.log('Saving tx info...');
await txsRepository.save(tx);
console.log('Done saving tx.');
}
async function getRelayers(): Promise<void> {
console.log('Getting relayer info...');
const relayerRepository = connection.getRepository(Relayer);
const relayerSource = new RelayerRegistrySource(RELAYER_REGISTRY_URL);
const relayersResp = await relayerSource.getRelayerInfoAsync();
const relayers = parseRelayers(relayersResp);
console.log('Saving relayer info...');
// for (const relayer of relayers) {
await relayerRepository.save(relayers);
// }
console.log('Done saving relayers.');
}

8
packages/pipeline/src/merge_v2_events.ts → packages/pipeline/src/scripts/merge_v2_events.ts
View File

@@ -2,10 +2,10 @@ import { web3Factory } from '@0x/dev-utils';
import 'reflect-metadata';
import { Connection, createConnection } from 'typeorm';
import { ExchangeEventsSource } from './data_sources/contract-wrappers/exchange_events';
import { ExchangeFillEvent } from './entities/ExchangeFillEvent';
import { deployConfig } from './ormconfig';
import { parseExchangeEvents } from './parsers/events';
import { ExchangeEventsSource } from '../data_sources/contract-wrappers/exchange_events';
import { ExchangeFillEvent } from '../entities/ExchangeFillEvent';
import { deployConfig } from '../ormconfig';
import { parseExchangeEvents } from '../parsers/events';
let connection: Connection;

60
packages/pipeline/src/scripts/pull_missing_events.ts Normal file
View File

@@ -0,0 +1,60 @@
import { web3Factory } from '@0x/dev-utils';
import { Web3ProviderEngine } from '@0x/subproviders';
import R = require('ramda');
import 'reflect-metadata';
import { Connection, createConnection, Repository } from 'typeorm';
import { ExchangeEventsSource } from '../data_sources/contract-wrappers/exchange_events';
import { ExchangeFillEvent } from '../entities/ExchangeFillEvent';
import { deployConfig } from '../ormconfig';
import { parseExchangeEvents } from '../parsers/events';
const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
const START_BLOCK_OFFSET = 1000; // Number of blocks before the last known block to consider when updating fill events.
const BATCH_SAVE_SIZE = 1000; // Number of events to save at once.
let connection: Connection;
(async () => {
connection = await createConnection(deployConfig);
const provider = web3Factory.getRpcProvider({
rpcUrl: 'https://mainnet.infura.io',
});
await getExchangeEventsAsync(provider);
process.exit(0);
})();
async function getExchangeEventsAsync(provider: Web3ProviderEngine): Promise<void> {
console.log('Checking existing event logs...');
const eventsRepository = connection.getRepository(ExchangeFillEvent);
const startBlock = await getStartBlockAsync(eventsRepository);
console.log(`Getting event logs starting at ${startBlock}...`);
const exchangeEvents = new ExchangeEventsSource(provider, 1);
const eventLogs = await exchangeEvents.getFillEventsAsync(startBlock);
console.log('Parsing events...');
const events = parseExchangeEvents(eventLogs);
console.log(`Retrieved and parsed ${events.length} total events.`);
console.log('Saving events...');
// Split the events into batches of size BATCH_SAVE_SIZE and save each batch
// in a single request. This reduces round-trip latency to the DB. We need
// to batch this way because saving an extremely large number of events in a
// single request causes problems.
for (const eventsBatch of R.splitEvery(BATCH_SAVE_SIZE, events)) {
await eventsRepository.save(eventsBatch);
}
const totalEvents = await eventsRepository.count();
console.log(`Done saving events. There are now ${totalEvents} total events.`);
}
async function getStartBlockAsync(eventsRepository: Repository<ExchangeFillEvent>): Promise<number> {
const fillEventCount = await eventsRepository.count();
if (fillEventCount === 0) {
console.log('No existing fill events found.');
return EXCHANGE_START_BLOCK;
}
const queryResult = await connection.query(
'SELECT "blockNumber" FROM exchange_fill_event ORDER BY "blockNumber" DESC LIMIT 1',
);
const lastKnownBlock = queryResult[0].blockNumber;
return lastKnownBlock - START_BLOCK_OFFSET;
}

31
packages/pipeline/src/scripts/update_relayer_info.ts Normal file
View File

@@ -0,0 +1,31 @@
import 'reflect-metadata';
import { Connection, createConnection } from 'typeorm';
import { RelayerRegistrySource } from '../data_sources/relayer-registry';
import { Relayer } from '../entities/Relayer';
import { deployConfig } from '../ormconfig';
import { parseRelayers } from '../parsers/relayer_registry';
// NOTE(albrow): We need to manually update this URL for now. Fix this when we
// have the relayer-registry behind semantic versioning.
const RELAYER_REGISTRY_URL =
'https://raw.githubusercontent.com/0xProject/0x-relayer-registry/4701c85677d161ea729a466aebbc1826c6aa2c0b/relayers.json';
let connection: Connection;
(async () => {
connection = await createConnection(deployConfig);
await getRelayers();
process.exit(0);
})();
async function getRelayers(): Promise<void> {
console.log('Getting latest relayer info...');
const relayerRepository = connection.getRepository(Relayer);
const relayerSource = new RelayerRegistrySource(RELAYER_REGISTRY_URL);
const relayersResp = await relayerSource.getRelayerInfoAsync();
const relayers = parseRelayers(relayersResp);
console.log('Saving relayer info...');
await relayerRepository.save(relayers);
console.log('Done saving relayer info.');
}