From 1991bd437f798e52d6210cf4fac173a10e30f0bd Mon Sep 17 00:00:00 2001 From: askeluv Date: Fri, 8 Mar 2019 12:32:21 +0800 Subject: [PATCH] First entity: Github repo --- .../1552016339539-CreateGithubRepo.ts | 33 +++++ .../pipeline/src/data_sources/github/index.ts | 37 ++++++ packages/pipeline/src/entities/github_repo.ts | 43 ++++++ packages/pipeline/src/entities/index.ts | 1 + packages/pipeline/src/ormconfig.ts | 2 + packages/pipeline/src/parsers/github/index.ts | 1 + packages/pipeline/src/parsers/github/repo.ts | 23 ++++ packages/pipeline/src/scripts/pull_github.ts | 28 ++++ .../test/fixtures/github/api_v3_repo.json | 122 ++++++++++++++++++ .../test/fixtures/github/api_v3_repo.ts | 21 +++ .../test/parsers/github/github_repo_test.ts | 25 ++++ packages/pipeline/tsconfig.json | 3 +- 12 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 packages/pipeline/migrations/1552016339539-CreateGithubRepo.ts create mode 100644 packages/pipeline/src/data_sources/github/index.ts create mode 100644 packages/pipeline/src/entities/github_repo.ts create mode 100644 packages/pipeline/src/parsers/github/index.ts create mode 100644 packages/pipeline/src/parsers/github/repo.ts create mode 100644 packages/pipeline/src/scripts/pull_github.ts create mode 100644 packages/pipeline/test/fixtures/github/api_v3_repo.json create mode 100644 packages/pipeline/test/fixtures/github/api_v3_repo.ts create mode 100644 packages/pipeline/test/parsers/github/github_repo_test.ts diff --git a/packages/pipeline/migrations/1552016339539-CreateGithubRepo.ts b/packages/pipeline/migrations/1552016339539-CreateGithubRepo.ts new file mode 100644 index 0000000000..84c9397267 --- /dev/null +++ b/packages/pipeline/migrations/1552016339539-CreateGithubRepo.ts @@ -0,0 +1,33 @@ +import {MigrationInterface, QueryRunner, Table} from "typeorm"; + +const table = new Table({ + name: 'raw.github_repo', + columns: [ + { name: 'observed_timestamp', type: 'numeric', isPrimary: true}, + { name: 'name', type: 'varchar', isPrimary: true }, + + { name: 'created_at', type: 'numeric', isNullable: false }, + { name: 'updated_at', type: 'numeric', isNullable: false }, + { name: 'pushed_at', type: 'numeric', isNullable: false }, + + { name: 'size', type: 'numeric', isNullable: false }, + { name: 'stargazers', type: 'numeric', isNullable: false }, + { name: 'watchers', type: 'numeric', isNullable: false }, + { name: 'forks', type: 'numeric', isNullable: false }, + { name: 'open_issues', type: 'numeric', isNullable: false }, + { name: 'network', type: 'numeric', isNullable: false }, + { name: 'subscribers', type: 'numeric', isNullable: false } + ], +}); + +export class CreateGithubRepo1552016339539 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable(table); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable(table); + } + +} diff --git a/packages/pipeline/src/data_sources/github/index.ts b/packages/pipeline/src/data_sources/github/index.ts new file mode 100644 index 0000000000..7d4f13f8b8 --- /dev/null +++ b/packages/pipeline/src/data_sources/github/index.ts @@ -0,0 +1,37 @@ +import { fetchAsync, logUtils } from '@0x/utils'; + +export interface GithubRepoResponse { + name: string; + created_at: string; + updated_at: string; + pushed_at: string; + size: number; + stargazers_count: number; + watchers_count: number; + forks: number; + open_issues: number; + network_count: number; + subscribers_count: number; +} + +// tslint:disable:prefer-function-over-method +// ^ Keep consistency with other sources and help logical organization +export class GithubSource { + public readonly _repoUrl: string; + public readonly _pullsUrl: string; + + constructor(owner: string, repo: string) { + const urlBase = 'https://api.github.com'; + this._repoUrl = `${urlBase}/repos/${owner}/${repo}`; + this._pullsUrl = `${urlBase}/repos/${owner}/${repo}/pulls`; + } + + /** + * Call Github API for repo and return result. + */ + public async getGithubRepoAsync(): Promise { + const resp = await fetchAsync(this._repoUrl); + const respJson: GithubRepoResponse = await resp.json(); + return respJson; + } +} diff --git a/packages/pipeline/src/entities/github_repo.ts b/packages/pipeline/src/entities/github_repo.ts new file mode 100644 index 0000000000..0010ca02df --- /dev/null +++ b/packages/pipeline/src/entities/github_repo.ts @@ -0,0 +1,43 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'github_repo', schema: 'raw' }) +export class GithubRepo { + @PrimaryColumn({ name: 'observed_timestamp', type: 'bigint', transformer: numberToBigIntTransformer }) + public observedTimestamp!: number; + + @PrimaryColumn({ name: 'name' }) + public name!: string; + + @Column({ name: 'created_at', type: 'bigint', transformer: numberToBigIntTransformer }) + public createdAt!: number; + + @Column({ name: 'updated_at', type: 'bigint', transformer: numberToBigIntTransformer }) + public updatedAt!: number; + + @Column({ name: 'pushed_at', type: 'bigint', transformer: numberToBigIntTransformer }) + public pushedAt!: number; + + @Column({ name: 'size', transformer: numberToBigIntTransformer }) + public size!: number; + + @Column({ name: 'stargazers', transformer: numberToBigIntTransformer }) + public stargazers!: number; + + @Column({ name: 'watchers', transformer: numberToBigIntTransformer }) + public watchers!: number; + + @Column({ name: 'forks', transformer: numberToBigIntTransformer }) + public forks!: number; + + @Column({ name: 'open_issues', transformer: numberToBigIntTransformer }) + public openIssues!: number; + + @Column({ name: 'network', transformer: numberToBigIntTransformer }) + public network!: number; + + @Column({ name: 'subscribers', transformer: numberToBigIntTransformer }) + public subscribers!: number; + +} diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts index 0f3e85f9c6..3f92ee528f 100644 --- a/packages/pipeline/src/entities/index.ts +++ b/packages/pipeline/src/entities/index.ts @@ -8,6 +8,7 @@ export { EtherscanTransaction } from './etherscan_transaction'; export { ExchangeCancelEvent } from './exchange_cancel_event'; export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event'; export { ExchangeFillEvent } from './exchange_fill_event'; +export { GithubRepo } from './github_repo'; export { NonfungibleDotComTrade } from './nonfungible_dot_com_trade'; export { OHLCVExternal } from './ohlcv_external'; export { Relayer } from './relayer'; diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index e2e6be7691..7410c67f28 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -13,6 +13,7 @@ import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, + GithubRepo, NonfungibleDotComTrade, OHLCVExternal, Relayer, @@ -37,6 +38,7 @@ const entities = [ ExchangeCancelUpToEvent, ExchangeFillEvent, ERC20ApprovalEvent, + GithubRepo, NonfungibleDotComTrade, OHLCVExternal, Relayer, diff --git a/packages/pipeline/src/parsers/github/index.ts b/packages/pipeline/src/parsers/github/index.ts new file mode 100644 index 0000000000..dfbac9d2a7 --- /dev/null +++ b/packages/pipeline/src/parsers/github/index.ts @@ -0,0 +1 @@ +export { parseGithubRepo } from './repo'; diff --git a/packages/pipeline/src/parsers/github/repo.ts b/packages/pipeline/src/parsers/github/repo.ts new file mode 100644 index 0000000000..e6b4d808be --- /dev/null +++ b/packages/pipeline/src/parsers/github/repo.ts @@ -0,0 +1,23 @@ +import { GithubRepoResponse } from '../../data_sources/github'; +import { GithubRepo } from '../../entities'; + +/** + * Converts a Github response from the API into an GithubRepo entity. + * @param rawRepo A Github response from the API into an GithubRepo entity. + */ +export function parseGithubRepo(rawRepo: GithubRepoResponse, observedTimestamp: number): GithubRepo { + const parsedRepo = new GithubRepo(); + parsedRepo.observedTimestamp = observedTimestamp; + parsedRepo.name = rawRepo.name; + parsedRepo.createdAt = Date.parse(rawRepo.created_at); + parsedRepo.updatedAt = Date.parse(rawRepo.updated_at); + parsedRepo.pushedAt = Date.parse(rawRepo.pushed_at); + parsedRepo.size = rawRepo.size; + parsedRepo.stargazers = rawRepo.stargazers_count; + parsedRepo.watchers = rawRepo.watchers_count; + parsedRepo.forks = rawRepo.forks; + parsedRepo.openIssues = rawRepo.open_issues; + parsedRepo.network = rawRepo.network_count; + parsedRepo.subscribers = rawRepo.subscribers_count; + return parsedRepo; +} diff --git a/packages/pipeline/src/scripts/pull_github.ts b/packages/pipeline/src/scripts/pull_github.ts new file mode 100644 index 0000000000..8aa0ae9a51 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_github.ts @@ -0,0 +1,28 @@ +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { GithubSource } from '../data_sources/github'; +import { GithubRepo } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseGithubRepo } from '../parsers/github'; +import { handleError } from '../utils'; + +const GITHUB_OWNER = '0xProject'; +const GITHUB_REPO = '0x-monorepo'; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + const GithubRepoRepository = connection.getRepository(GithubRepo); + const githubSource = new GithubSource(GITHUB_OWNER, GITHUB_REPO); + + // get repo and save + const rawRepo = await githubSource.getGithubRepoAsync(); + const observedTimestamp = Date.now(); + const record = parseGithubRepo(rawRepo, observedTimestamp); + await GithubRepoRepository.save(record); + + // TODO: get pull requests and save + + process.exit(0); +})().catch(handleError); diff --git a/packages/pipeline/test/fixtures/github/api_v3_repo.json b/packages/pipeline/test/fixtures/github/api_v3_repo.json new file mode 100644 index 0000000000..37396d34fc --- /dev/null +++ b/packages/pipeline/test/fixtures/github/api_v3_repo.json @@ -0,0 +1,122 @@ +{ + "id": 92181371, + "node_id": "MDEwOlJlcG9zaXRvcnk5MjE4MTM3MQ==", + "name": "0x-monorepo", + "full_name": "0xProject/0x-monorepo", + "private": false, + "owner": { + "login": "0xProject", + "id": 24832717, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjI0ODMyNzE3", + "avatar_url": "https://avatars2.githubusercontent.com/u/24832717?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/0xProject", + "html_url": "https://github.com/0xProject", + "followers_url": "https://api.github.com/users/0xProject/followers", + "following_url": "https://api.github.com/users/0xProject/following{/other_user}", + "gists_url": "https://api.github.com/users/0xProject/gists{/gist_id}", + "starred_url": "https://api.github.com/users/0xProject/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/0xProject/subscriptions", + "organizations_url": "https://api.github.com/users/0xProject/orgs", + "repos_url": "https://api.github.com/users/0xProject/repos", + "events_url": "https://api.github.com/users/0xProject/events{/privacy}", + "received_events_url": "https://api.github.com/users/0xProject/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/0xProject/0x-monorepo", + "description": "0x protocol monorepo - includes our smart contracts and many developer tools", + "fork": false, + "url": "https://api.github.com/repos/0xProject/0x-monorepo", + "forks_url": "https://api.github.com/repos/0xProject/0x-monorepo/forks", + "keys_url": "https://api.github.com/repos/0xProject/0x-monorepo/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/0xProject/0x-monorepo/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/0xProject/0x-monorepo/teams", + "hooks_url": "https://api.github.com/repos/0xProject/0x-monorepo/hooks", + "issue_events_url": "https://api.github.com/repos/0xProject/0x-monorepo/issues/events{/number}", + "events_url": "https://api.github.com/repos/0xProject/0x-monorepo/events", + "assignees_url": "https://api.github.com/repos/0xProject/0x-monorepo/assignees{/user}", + "branches_url": "https://api.github.com/repos/0xProject/0x-monorepo/branches{/branch}", + "tags_url": "https://api.github.com/repos/0xProject/0x-monorepo/tags", + "blobs_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/0xProject/0x-monorepo/statuses/{sha}", + "languages_url": "https://api.github.com/repos/0xProject/0x-monorepo/languages", + "stargazers_url": "https://api.github.com/repos/0xProject/0x-monorepo/stargazers", + "contributors_url": "https://api.github.com/repos/0xProject/0x-monorepo/contributors", + "subscribers_url": "https://api.github.com/repos/0xProject/0x-monorepo/subscribers", + "subscription_url": "https://api.github.com/repos/0xProject/0x-monorepo/subscription", + "commits_url": "https://api.github.com/repos/0xProject/0x-monorepo/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/0xProject/0x-monorepo/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/0xProject/0x-monorepo/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/0xProject/0x-monorepo/contents/{+path}", + "compare_url": "https://api.github.com/repos/0xProject/0x-monorepo/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/0xProject/0x-monorepo/merges", + "archive_url": "https://api.github.com/repos/0xProject/0x-monorepo/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/0xProject/0x-monorepo/downloads", + "issues_url": "https://api.github.com/repos/0xProject/0x-monorepo/issues{/number}", + "pulls_url": "https://api.github.com/repos/0xProject/0x-monorepo/pulls{/number}", + "milestones_url": "https://api.github.com/repos/0xProject/0x-monorepo/milestones{/number}", + "notifications_url": "https://api.github.com/repos/0xProject/0x-monorepo/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/0xProject/0x-monorepo/labels{/name}", + "releases_url": "https://api.github.com/repos/0xProject/0x-monorepo/releases{/id}", + "deployments_url": "https://api.github.com/repos/0xProject/0x-monorepo/deployments", + "created_at": "2017-05-23T14:17:33Z", + "updated_at": "2019-03-06T21:48:49Z", + "pushed_at": "2019-03-06T23:59:05Z", + "git_url": "git://github.com/0xProject/0x-monorepo.git", + "ssh_url": "git@github.com:0xProject/0x-monorepo.git", + "clone_url": "https://github.com/0xProject/0x-monorepo.git", + "svn_url": "https://github.com/0xProject/0x-monorepo", + "homepage": "", + "size": 86538, + "stargazers_count": 989, + "watchers_count": 989, + "language": "TypeScript", + "has_issues": true, + "has_projects": false, + "has_downloads": true, + "has_wiki": false, + "has_pages": false, + "forks_count": 294, + "mirror_url": null, + "archived": false, + "open_issues_count": 46, + "license": { + "key": "other", + "name": "Other", + "spdx_id": "NOASSERTION", + "url": null, + "node_id": "MDc6TGljZW5zZTA=" + }, + "forks": 294, + "open_issues": 46, + "watchers": 989, + "default_branch": "development", + "organization": { + "login": "0xProject", + "id": 24832717, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjI0ODMyNzE3", + "avatar_url": "https://avatars2.githubusercontent.com/u/24832717?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/0xProject", + "html_url": "https://github.com/0xProject", + "followers_url": "https://api.github.com/users/0xProject/followers", + "following_url": "https://api.github.com/users/0xProject/following{/other_user}", + "gists_url": "https://api.github.com/users/0xProject/gists{/gist_id}", + "starred_url": "https://api.github.com/users/0xProject/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/0xProject/subscriptions", + "organizations_url": "https://api.github.com/users/0xProject/orgs", + "repos_url": "https://api.github.com/users/0xProject/repos", + "events_url": "https://api.github.com/users/0xProject/events{/privacy}", + "received_events_url": "https://api.github.com/users/0xProject/received_events", + "type": "Organization", + "site_admin": false + }, + "network_count": 294, + "subscribers_count": 89 + } + \ No newline at end of file diff --git a/packages/pipeline/test/fixtures/github/api_v3_repo.ts b/packages/pipeline/test/fixtures/github/api_v3_repo.ts new file mode 100644 index 0000000000..3a6d9f9e4b --- /dev/null +++ b/packages/pipeline/test/fixtures/github/api_v3_repo.ts @@ -0,0 +1,21 @@ +import { GithubRepo } from '../../../src/entities'; + +// To re-create the JSON file from the API (e.g. if the API output schema changes), run the below command: +// curl https://api.github.com/repos/0xProject/0x-monorepo +// docs here: https://developer.github.com/v3/repos/#get + +const ParsedGithubRepo: GithubRepo = { + observedTimestamp: Date.now(), + name: '0x-monorepo', + createdAt: 1495549053000, + updatedAt: 1551908929000, + pushedAt: 1551916745000, + size: 86538, + stargazers: 989, + watchers: 989, + forks: 294, + openIssues: 46, + network: 294, + subscribers: 89, + }; +export { ParsedGithubRepo }; diff --git a/packages/pipeline/test/parsers/github/github_repo_test.ts b/packages/pipeline/test/parsers/github/github_repo_test.ts new file mode 100644 index 0000000000..8e6a9c0d72 --- /dev/null +++ b/packages/pipeline/test/parsers/github/github_repo_test.ts @@ -0,0 +1,25 @@ +import * as chai from 'chai'; +import 'mocha'; + +import { GithubRepoResponse } from '../../../src/data_sources/github'; +import { parseGithubRepo } from '../../../src/parsers/github'; +import { chaiSetup } from '../../utils/chai_setup'; + +import { ParsedGithubRepo } from '../../fixtures/github/api_v3_repo'; +import * as githubRepoResponse from '../../fixtures/github/api_v3_repo.json'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('github_repo', () => { + describe('parseGithubRepo', () => { + it('converts GithubResponse to GithubRepo entities', () => { + const response: GithubRepoResponse = githubRepoResponse; + const expected = ParsedGithubRepo; + const observedTimestamp = expected.observedTimestamp; + const actual = parseGithubRepo(response, observedTimestamp); + expect(actual).deep.equal(expected); + }); + }); +}); diff --git a/packages/pipeline/tsconfig.json b/packages/pipeline/tsconfig.json index 4c43775d74..d4db6fafa3 100644 --- a/packages/pipeline/tsconfig.json +++ b/packages/pipeline/tsconfig.json @@ -14,6 +14,7 @@ "./test/fixtures/copper/api_v1_list_activities.json", "./test/fixtures/copper/api_v1_list_leads.json", "./test/fixtures/copper/api_v1_list_opportunities.json", - "./test/fixtures/etherscan/api_v1_accounts_transactions.json" + "./test/fixtures/etherscan/api_v1_accounts_transactions.json", + "./test/fixtures/github/api_v3_repo.json" ] }