First entity: Github repo
This commit is contained in:
askeluv
parent
2c329023c2
commit
1991bd437f
@ -0,0 +1,33 @@
|
||||
import {MigrationInterface, QueryRunner, Table} from "typeorm";
|
||||
|
||||
const table = new Table({
|
||||
name: 'raw.github_repo',
|
||||
columns: [
|
||||
{ name: 'observed_timestamp', type: 'numeric', isPrimary: true},
|
||||
{ name: 'name', type: 'varchar', isPrimary: true },
|
||||
|
||||
{ name: 'created_at', type: 'numeric', isNullable: false },
|
||||
{ name: 'updated_at', type: 'numeric', isNullable: false },
|
||||
{ name: 'pushed_at', type: 'numeric', isNullable: false },
|
||||
|
||||
{ name: 'size', type: 'numeric', isNullable: false },
|
||||
{ name: 'stargazers', type: 'numeric', isNullable: false },
|
||||
{ name: 'watchers', type: 'numeric', isNullable: false },
|
||||
{ name: 'forks', type: 'numeric', isNullable: false },
|
||||
{ name: 'open_issues', type: 'numeric', isNullable: false },
|
||||
{ name: 'network', type: 'numeric', isNullable: false },
|
||||
{ name: 'subscribers', type: 'numeric', isNullable: false }
|
||||
],
|
||||
});
|
||||
|
||||
export class CreateGithubRepo1552016339539 implements MigrationInterface {
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<any> {
|
||||
await queryRunner.createTable(table);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<any> {
|
||||
await queryRunner.dropTable(table);
|
||||
}
|
||||
|
||||
}
|
||||
37
packages/pipeline/src/data_sources/github/index.ts
Normal file
37
packages/pipeline/src/data_sources/github/index.ts
Normal file
@ -0,0 +1,37 @@
|
||||
import { fetchAsync, logUtils } from '@0x/utils';
|
||||
|
||||
export interface GithubRepoResponse {
|
||||
name: string;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
pushed_at: string;
|
||||
size: number;
|
||||
stargazers_count: number;
|
||||
watchers_count: number;
|
||||
forks: number;
|
||||
open_issues: number;
|
||||
network_count: number;
|
||||
subscribers_count: number;
|
||||
}
|
||||
|
||||
// tslint:disable:prefer-function-over-method
|
||||
// ^ Keep consistency with other sources and help logical organization
|
||||
export class GithubSource {
|
||||
public readonly _repoUrl: string;
|
||||
public readonly _pullsUrl: string;
|
||||
|
||||
constructor(owner: string, repo: string) {
|
||||
const urlBase = 'https://api.github.com';
|
||||
this._repoUrl = `${urlBase}/repos/${owner}/${repo}`;
|
||||
this._pullsUrl = `${urlBase}/repos/${owner}/${repo}/pulls`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Call Github API for repo and return result.
|
||||
*/
|
||||
public async getGithubRepoAsync(): Promise<GithubRepoResponse> {
|
||||
const resp = await fetchAsync(this._repoUrl);
|
||||
const respJson: GithubRepoResponse = await resp.json();
|
||||
return respJson;
|
||||
}
|
||||
}
|
||||
43
packages/pipeline/src/entities/github_repo.ts
Normal file
43
packages/pipeline/src/entities/github_repo.ts
Normal file
@ -0,0 +1,43 @@
|
||||
import { Column, Entity, PrimaryColumn } from 'typeorm';
|
||||
|
||||
import { numberToBigIntTransformer } from '../utils';
|
||||
|
||||
@Entity({ name: 'github_repo', schema: 'raw' })
|
||||
export class GithubRepo {
|
||||
@PrimaryColumn({ name: 'observed_timestamp', type: 'bigint', transformer: numberToBigIntTransformer })
|
||||
public observedTimestamp!: number;
|
||||
|
||||
@PrimaryColumn({ name: 'name' })
|
||||
public name!: string;
|
||||
|
||||
@Column({ name: 'created_at', type: 'bigint', transformer: numberToBigIntTransformer })
|
||||
public createdAt!: number;
|
||||
|
||||
@Column({ name: 'updated_at', type: 'bigint', transformer: numberToBigIntTransformer })
|
||||
public updatedAt!: number;
|
||||
|
||||
@Column({ name: 'pushed_at', type: 'bigint', transformer: numberToBigIntTransformer })
|
||||
public pushedAt!: number;
|
||||
|
||||
@Column({ name: 'size', transformer: numberToBigIntTransformer })
|
||||
public size!: number;
|
||||
|
||||
@Column({ name: 'stargazers', transformer: numberToBigIntTransformer })
|
||||
public stargazers!: number;
|
||||
|
||||
@Column({ name: 'watchers', transformer: numberToBigIntTransformer })
|
||||
public watchers!: number;
|
||||
|
||||
@Column({ name: 'forks', transformer: numberToBigIntTransformer })
|
||||
public forks!: number;
|
||||
|
||||
@Column({ name: 'open_issues', transformer: numberToBigIntTransformer })
|
||||
public openIssues!: number;
|
||||
|
||||
@Column({ name: 'network', transformer: numberToBigIntTransformer })
|
||||
public network!: number;
|
||||
|
||||
@Column({ name: 'subscribers', transformer: numberToBigIntTransformer })
|
||||
public subscribers!: number;
|
||||
|
||||
}
|
||||
@ -8,6 +8,7 @@ export { EtherscanTransaction } from './etherscan_transaction';
|
||||
export { ExchangeCancelEvent } from './exchange_cancel_event';
|
||||
export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event';
|
||||
export { ExchangeFillEvent } from './exchange_fill_event';
|
||||
export { GithubRepo } from './github_repo';
|
||||
export { NonfungibleDotComTrade } from './nonfungible_dot_com_trade';
|
||||
export { OHLCVExternal } from './ohlcv_external';
|
||||
export { Relayer } from './relayer';
|
||||
|
||||
@ -13,6 +13,7 @@ import {
|
||||
ExchangeCancelEvent,
|
||||
ExchangeCancelUpToEvent,
|
||||
ExchangeFillEvent,
|
||||
GithubRepo,
|
||||
NonfungibleDotComTrade,
|
||||
OHLCVExternal,
|
||||
Relayer,
|
||||
@ -37,6 +38,7 @@ const entities = [
|
||||
ExchangeCancelUpToEvent,
|
||||
ExchangeFillEvent,
|
||||
ERC20ApprovalEvent,
|
||||
GithubRepo,
|
||||
NonfungibleDotComTrade,
|
||||
OHLCVExternal,
|
||||
Relayer,
|
||||
|
||||
1
packages/pipeline/src/parsers/github/index.ts
Normal file
1
packages/pipeline/src/parsers/github/index.ts
Normal file
@ -0,0 +1 @@
|
||||
export { parseGithubRepo } from './repo';
|
||||
23
packages/pipeline/src/parsers/github/repo.ts
Normal file
23
packages/pipeline/src/parsers/github/repo.ts
Normal file
@ -0,0 +1,23 @@
|
||||
import { GithubRepoResponse } from '../../data_sources/github';
|
||||
import { GithubRepo } from '../../entities';
|
||||
|
||||
/**
|
||||
* Converts a Github response from the API into an GithubRepo entity.
|
||||
* @param rawRepo A Github response from the API into an GithubRepo entity.
|
||||
*/
|
||||
export function parseGithubRepo(rawRepo: GithubRepoResponse, observedTimestamp: number): GithubRepo {
|
||||
const parsedRepo = new GithubRepo();
|
||||
parsedRepo.observedTimestamp = observedTimestamp;
|
||||
parsedRepo.name = rawRepo.name;
|
||||
parsedRepo.createdAt = Date.parse(rawRepo.created_at);
|
||||
parsedRepo.updatedAt = Date.parse(rawRepo.updated_at);
|
||||
parsedRepo.pushedAt = Date.parse(rawRepo.pushed_at);
|
||||
parsedRepo.size = rawRepo.size;
|
||||
parsedRepo.stargazers = rawRepo.stargazers_count;
|
||||
parsedRepo.watchers = rawRepo.watchers_count;
|
||||
parsedRepo.forks = rawRepo.forks;
|
||||
parsedRepo.openIssues = rawRepo.open_issues;
|
||||
parsedRepo.network = rawRepo.network_count;
|
||||
parsedRepo.subscribers = rawRepo.subscribers_count;
|
||||
return parsedRepo;
|
||||
}
|
||||
28
packages/pipeline/src/scripts/pull_github.ts
Normal file
28
packages/pipeline/src/scripts/pull_github.ts
Normal file
@ -0,0 +1,28 @@
|
||||
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
|
||||
|
||||
import { GithubSource } from '../data_sources/github';
|
||||
import { GithubRepo } from '../entities';
|
||||
import * as ormConfig from '../ormconfig';
|
||||
import { parseGithubRepo } from '../parsers/github';
|
||||
import { handleError } from '../utils';
|
||||
|
||||
const GITHUB_OWNER = '0xProject';
|
||||
const GITHUB_REPO = '0x-monorepo';
|
||||
|
||||
let connection: Connection;
|
||||
|
||||
(async () => {
|
||||
connection = await createConnection(ormConfig as ConnectionOptions);
|
||||
const GithubRepoRepository = connection.getRepository(GithubRepo);
|
||||
const githubSource = new GithubSource(GITHUB_OWNER, GITHUB_REPO);
|
||||
|
||||
// get repo and save
|
||||
const rawRepo = await githubSource.getGithubRepoAsync();
|
||||
const observedTimestamp = Date.now();
|
||||
const record = parseGithubRepo(rawRepo, observedTimestamp);
|
||||
await GithubRepoRepository.save(record);
|
||||
|
||||
// TODO: get pull requests and save
|
||||
|
||||
process.exit(0);
|
||||
})().catch(handleError);
|
||||
122
packages/pipeline/test/fixtures/github/api_v3_repo.json
vendored
Normal file
122
packages/pipeline/test/fixtures/github/api_v3_repo.json
vendored
Normal file
@ -0,0 +1,122 @@
|
||||
{
|
||||
"id": 92181371,
|
||||
"node_id": "MDEwOlJlcG9zaXRvcnk5MjE4MTM3MQ==",
|
||||
"name": "0x-monorepo",
|
||||
"full_name": "0xProject/0x-monorepo",
|
||||
"private": false,
|
||||
"owner": {
|
||||
"login": "0xProject",
|
||||
"id": 24832717,
|
||||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjI0ODMyNzE3",
|
||||
"avatar_url": "https://avatars2.githubusercontent.com/u/24832717?v=4",
|
||||
"gravatar_id": "",
|
||||
"url": "https://api.github.com/users/0xProject",
|
||||
"html_url": "https://github.com/0xProject",
|
||||
"followers_url": "https://api.github.com/users/0xProject/followers",
|
||||
"following_url": "https://api.github.com/users/0xProject/following{/other_user}",
|
||||
"gists_url": "https://api.github.com/users/0xProject/gists{/gist_id}",
|
||||
"starred_url": "https://api.github.com/users/0xProject/starred{/owner}{/repo}",
|
||||
"subscriptions_url": "https://api.github.com/users/0xProject/subscriptions",
|
||||
"organizations_url": "https://api.github.com/users/0xProject/orgs",
|
||||
"repos_url": "https://api.github.com/users/0xProject/repos",
|
||||
"events_url": "https://api.github.com/users/0xProject/events{/privacy}",
|
||||
"received_events_url": "https://api.github.com/users/0xProject/received_events",
|
||||
"type": "Organization",
|
||||
"site_admin": false
|
||||
},
|
||||
"html_url": "https://github.com/0xProject/0x-monorepo",
|
||||
"description": "0x protocol monorepo - includes our smart contracts and many developer tools",
|
||||
"fork": false,
|
||||
"url": "https://api.github.com/repos/0xProject/0x-monorepo",
|
||||
"forks_url": "https://api.github.com/repos/0xProject/0x-monorepo/forks",
|
||||
"keys_url": "https://api.github.com/repos/0xProject/0x-monorepo/keys{/key_id}",
|
||||
"collaborators_url": "https://api.github.com/repos/0xProject/0x-monorepo/collaborators{/collaborator}",
|
||||
"teams_url": "https://api.github.com/repos/0xProject/0x-monorepo/teams",
|
||||
"hooks_url": "https://api.github.com/repos/0xProject/0x-monorepo/hooks",
|
||||
"issue_events_url": "https://api.github.com/repos/0xProject/0x-monorepo/issues/events{/number}",
|
||||
"events_url": "https://api.github.com/repos/0xProject/0x-monorepo/events",
|
||||
"assignees_url": "https://api.github.com/repos/0xProject/0x-monorepo/assignees{/user}",
|
||||
"branches_url": "https://api.github.com/repos/0xProject/0x-monorepo/branches{/branch}",
|
||||
"tags_url": "https://api.github.com/repos/0xProject/0x-monorepo/tags",
|
||||
"blobs_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/blobs{/sha}",
|
||||
"git_tags_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/tags{/sha}",
|
||||
"git_refs_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/refs{/sha}",
|
||||
"trees_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/trees{/sha}",
|
||||
"statuses_url": "https://api.github.com/repos/0xProject/0x-monorepo/statuses/{sha}",
|
||||
"languages_url": "https://api.github.com/repos/0xProject/0x-monorepo/languages",
|
||||
"stargazers_url": "https://api.github.com/repos/0xProject/0x-monorepo/stargazers",
|
||||
"contributors_url": "https://api.github.com/repos/0xProject/0x-monorepo/contributors",
|
||||
"subscribers_url": "https://api.github.com/repos/0xProject/0x-monorepo/subscribers",
|
||||
"subscription_url": "https://api.github.com/repos/0xProject/0x-monorepo/subscription",
|
||||
"commits_url": "https://api.github.com/repos/0xProject/0x-monorepo/commits{/sha}",
|
||||
"git_commits_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/commits{/sha}",
|
||||
"comments_url": "https://api.github.com/repos/0xProject/0x-monorepo/comments{/number}",
|
||||
"issue_comment_url": "https://api.github.com/repos/0xProject/0x-monorepo/issues/comments{/number}",
|
||||
"contents_url": "https://api.github.com/repos/0xProject/0x-monorepo/contents/{+path}",
|
||||
"compare_url": "https://api.github.com/repos/0xProject/0x-monorepo/compare/{base}...{head}",
|
||||
"merges_url": "https://api.github.com/repos/0xProject/0x-monorepo/merges",
|
||||
"archive_url": "https://api.github.com/repos/0xProject/0x-monorepo/{archive_format}{/ref}",
|
||||
"downloads_url": "https://api.github.com/repos/0xProject/0x-monorepo/downloads",
|
||||
"issues_url": "https://api.github.com/repos/0xProject/0x-monorepo/issues{/number}",
|
||||
"pulls_url": "https://api.github.com/repos/0xProject/0x-monorepo/pulls{/number}",
|
||||
"milestones_url": "https://api.github.com/repos/0xProject/0x-monorepo/milestones{/number}",
|
||||
"notifications_url": "https://api.github.com/repos/0xProject/0x-monorepo/notifications{?since,all,participating}",
|
||||
"labels_url": "https://api.github.com/repos/0xProject/0x-monorepo/labels{/name}",
|
||||
"releases_url": "https://api.github.com/repos/0xProject/0x-monorepo/releases{/id}",
|
||||
"deployments_url": "https://api.github.com/repos/0xProject/0x-monorepo/deployments",
|
||||
"created_at": "2017-05-23T14:17:33Z",
|
||||
"updated_at": "2019-03-06T21:48:49Z",
|
||||
"pushed_at": "2019-03-06T23:59:05Z",
|
||||
"git_url": "git://github.com/0xProject/0x-monorepo.git",
|
||||
"ssh_url": "git@github.com:0xProject/0x-monorepo.git",
|
||||
"clone_url": "https://github.com/0xProject/0x-monorepo.git",
|
||||
"svn_url": "https://github.com/0xProject/0x-monorepo",
|
||||
"homepage": "",
|
||||
"size": 86538,
|
||||
"stargazers_count": 989,
|
||||
"watchers_count": 989,
|
||||
"language": "TypeScript",
|
||||
"has_issues": true,
|
||||
"has_projects": false,
|
||||
"has_downloads": true,
|
||||
"has_wiki": false,
|
||||
"has_pages": false,
|
||||
"forks_count": 294,
|
||||
"mirror_url": null,
|
||||
"archived": false,
|
||||
"open_issues_count": 46,
|
||||
"license": {
|
||||
"key": "other",
|
||||
"name": "Other",
|
||||
"spdx_id": "NOASSERTION",
|
||||
"url": null,
|
||||
"node_id": "MDc6TGljZW5zZTA="
|
||||
},
|
||||
"forks": 294,
|
||||
"open_issues": 46,
|
||||
"watchers": 989,
|
||||
"default_branch": "development",
|
||||
"organization": {
|
||||
"login": "0xProject",
|
||||
"id": 24832717,
|
||||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjI0ODMyNzE3",
|
||||
"avatar_url": "https://avatars2.githubusercontent.com/u/24832717?v=4",
|
||||
"gravatar_id": "",
|
||||
"url": "https://api.github.com/users/0xProject",
|
||||
"html_url": "https://github.com/0xProject",
|
||||
"followers_url": "https://api.github.com/users/0xProject/followers",
|
||||
"following_url": "https://api.github.com/users/0xProject/following{/other_user}",
|
||||
"gists_url": "https://api.github.com/users/0xProject/gists{/gist_id}",
|
||||
"starred_url": "https://api.github.com/users/0xProject/starred{/owner}{/repo}",
|
||||
"subscriptions_url": "https://api.github.com/users/0xProject/subscriptions",
|
||||
"organizations_url": "https://api.github.com/users/0xProject/orgs",
|
||||
"repos_url": "https://api.github.com/users/0xProject/repos",
|
||||
"events_url": "https://api.github.com/users/0xProject/events{/privacy}",
|
||||
"received_events_url": "https://api.github.com/users/0xProject/received_events",
|
||||
"type": "Organization",
|
||||
"site_admin": false
|
||||
},
|
||||
"network_count": 294,
|
||||
"subscribers_count": 89
|
||||
}
|
||||
|
||||
21
packages/pipeline/test/fixtures/github/api_v3_repo.ts
vendored
Normal file
21
packages/pipeline/test/fixtures/github/api_v3_repo.ts
vendored
Normal file
@ -0,0 +1,21 @@
|
||||
import { GithubRepo } from '../../../src/entities';
|
||||
|
||||
// To re-create the JSON file from the API (e.g. if the API output schema changes), run the below command:
|
||||
// curl https://api.github.com/repos/0xProject/0x-monorepo
|
||||
// docs here: https://developer.github.com/v3/repos/#get
|
||||
|
||||
const ParsedGithubRepo: GithubRepo = {
|
||||
observedTimestamp: Date.now(),
|
||||
name: '0x-monorepo',
|
||||
createdAt: 1495549053000,
|
||||
updatedAt: 1551908929000,
|
||||
pushedAt: 1551916745000,
|
||||
size: 86538,
|
||||
stargazers: 989,
|
||||
watchers: 989,
|
||||
forks: 294,
|
||||
openIssues: 46,
|
||||
network: 294,
|
||||
subscribers: 89,
|
||||
};
|
||||
export { ParsedGithubRepo };
|
||||
25
packages/pipeline/test/parsers/github/github_repo_test.ts
Normal file
25
packages/pipeline/test/parsers/github/github_repo_test.ts
Normal file
@ -0,0 +1,25 @@
|
||||
import * as chai from 'chai';
|
||||
import 'mocha';
|
||||
|
||||
import { GithubRepoResponse } from '../../../src/data_sources/github';
|
||||
import { parseGithubRepo } from '../../../src/parsers/github';
|
||||
import { chaiSetup } from '../../utils/chai_setup';
|
||||
|
||||
import { ParsedGithubRepo } from '../../fixtures/github/api_v3_repo';
|
||||
import * as githubRepoResponse from '../../fixtures/github/api_v3_repo.json';
|
||||
|
||||
chaiSetup.configure();
|
||||
const expect = chai.expect;
|
||||
|
||||
// tslint:disable:custom-no-magic-numbers
|
||||
describe('github_repo', () => {
|
||||
describe('parseGithubRepo', () => {
|
||||
it('converts GithubResponse to GithubRepo entities', () => {
|
||||
const response: GithubRepoResponse = githubRepoResponse;
|
||||
const expected = ParsedGithubRepo;
|
||||
const observedTimestamp = expected.observedTimestamp;
|
||||
const actual = parseGithubRepo(response, observedTimestamp);
|
||||
expect(actual).deep.equal(expected);
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -14,6 +14,7 @@
|
||||
"./test/fixtures/copper/api_v1_list_activities.json",
|
||||
"./test/fixtures/copper/api_v1_list_leads.json",
|
||||
"./test/fixtures/copper/api_v1_list_opportunities.json",
|
||||
"./test/fixtures/etherscan/api_v1_accounts_transactions.json"
|
||||
"./test/fixtures/etherscan/api_v1_accounts_transactions.json",
|
||||
"./test/fixtures/github/api_v3_repo.json"
|
||||
]
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user