crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Second entity: Github pull request

Browse Source
This commit is contained in:
askeluv
2019-03-08 14:37:36 +08:00
parent 1991bd437f
commit 55fd71c5e1
13 changed files with 532 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
packages/pipeline/src/scripts/pull_github.ts
View File

@@ -1,28 +1,45 @@
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import { logUtils } from '@0x/utils';
import { GithubSource } from '../data_sources/github';
import { GithubRepo } from '../entities';
import { GithubPullRequest, GithubRepo } from '../entities';
import * as ormConfig from '../ormconfig';
import { parseGithubRepo } from '../parsers/github';
import { parseGithubPulls, parseGithubRepo } from '../parsers/github';
import { handleError } from '../utils';
const GITHUB_OWNER = '0xProject';
const GITHUB_REPO = '0x-monorepo';
const RECORDS_PER_PAGE = 100;
let connection: Connection;
(async () => {
connection = await createConnection(ormConfig as ConnectionOptions);
const GithubRepoRepository = connection.getRepository(GithubRepo);
const githubSource = new GithubSource(GITHUB_OWNER, GITHUB_REPO);
const observedTimestamp = Date.now();
// get repo and save
const GithubRepoRepository = connection.getRepository(GithubRepo);
logUtils.log('Fetching Github repo from API.');
const rawRepo = await githubSource.getGithubRepoAsync();
const observedTimestamp = Date.now();
const record = parseGithubRepo(rawRepo, observedTimestamp);
await GithubRepoRepository.save(record);
const repo = parseGithubRepo(rawRepo, observedTimestamp);
logUtils.log('Saving Github repo to database.');
await GithubRepoRepository.save(repo);
// TODO: get pull requests and save
// get pull requests and save
const GithubPullRequestRepository = connection.getRepository(GithubPullRequest);
let numberOfRecords = RECORDS_PER_PAGE;
let page = 1;
while (numberOfRecords === RECORDS_PER_PAGE) {
logUtils.log(`Fetching Github pull requests from API, page: ${page}.`);
const rawPulls = await githubSource.getGithubPullsAsync(page);
const pulls = parseGithubPulls(rawPulls, observedTimestamp);
numberOfRecords = pulls.length;
page++;
logUtils.log(`Saving ${pulls.length} pull requests to database.`);
await GithubPullRequestRepository.save(pulls);
}
process.exit(0);
})().catch(handleError);