First entity: Github repo

packages/pipeline/migrations/1552016339539-CreateGithubRepo.ts

@@ -0,0 +1,33 @@
+import {MigrationInterface, QueryRunner, Table} from "typeorm";
+
+const table = new Table({
+    name: 'raw.github_repo',
+    columns: [
+        { name: 'observed_timestamp', type: 'numeric', isPrimary: true},
+        { name: 'name', type: 'varchar', isPrimary: true },
+
+        { name: 'created_at', type: 'numeric', isNullable: false },
+        { name: 'updated_at', type: 'numeric', isNullable: false },
+        { name: 'pushed_at', type: 'numeric', isNullable: false },
+
+        { name: 'size', type: 'numeric', isNullable: false },
+        { name: 'stargazers', type: 'numeric', isNullable: false },
+        { name: 'watchers', type: 'numeric', isNullable: false },
+        { name: 'forks', type: 'numeric', isNullable: false },
+        { name: 'open_issues', type: 'numeric', isNullable: false },
+        { name: 'network', type: 'numeric', isNullable: false },
+        { name: 'subscribers', type: 'numeric', isNullable: false }        
+    ],
+});
+
+export class CreateGithubRepo1552016339539 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.createTable(table);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.dropTable(table);
+    }
+
+}

packages/pipeline/src/data_sources/github/index.ts

@@ -0,0 +1,37 @@
+import { fetchAsync, logUtils } from '@0x/utils';
+
+export interface GithubRepoResponse {
+    name: string;
+    created_at: string;
+    updated_at: string;
+    pushed_at: string;
+    size: number;
+    stargazers_count: number;
+    watchers_count: number;
+    forks: number;
+    open_issues: number;
+    network_count: number;
+    subscribers_count: number;
+}
+
+// tslint:disable:prefer-function-over-method
+// ^ Keep consistency with other sources and help logical organization
+export class GithubSource {
+    public readonly _repoUrl: string;
+    public readonly _pullsUrl: string;
+
+    constructor(owner: string, repo: string) {
+        const urlBase = 'https://api.github.com';
+        this._repoUrl = `${urlBase}/repos/${owner}/${repo}`;
+        this._pullsUrl = `${urlBase}/repos/${owner}/${repo}/pulls`;
+    }
+
+    /**
+     * Call Github API for repo and return result.
+     */
+    public async getGithubRepoAsync(): Promise<GithubRepoResponse> {
+        const resp = await fetchAsync(this._repoUrl);
+        const respJson: GithubRepoResponse = await resp.json();
+        return respJson;
+    }
+}

packages/pipeline/src/entities/github_repo.ts

@@ -0,0 +1,43 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'github_repo', schema: 'raw' })
+export class GithubRepo {
+    @PrimaryColumn({ name: 'observed_timestamp', type: 'bigint', transformer: numberToBigIntTransformer })
+    public observedTimestamp!: number;
+
+    @PrimaryColumn({ name: 'name' })
+    public name!: string;
+
+    @Column({ name: 'created_at', type: 'bigint', transformer: numberToBigIntTransformer })
+    public createdAt!: number;
+
+    @Column({ name: 'updated_at', type: 'bigint', transformer: numberToBigIntTransformer })
+    public updatedAt!: number;
+
+    @Column({ name: 'pushed_at', type: 'bigint', transformer: numberToBigIntTransformer })
+    public pushedAt!: number;
+
+    @Column({ name: 'size', transformer: numberToBigIntTransformer })
+    public size!: number;
+
+    @Column({ name: 'stargazers', transformer: numberToBigIntTransformer })
+    public stargazers!: number;
+
+    @Column({ name: 'watchers', transformer: numberToBigIntTransformer })
+    public watchers!: number;
+
+    @Column({ name: 'forks', transformer: numberToBigIntTransformer })
+    public forks!: number;
+
+    @Column({ name: 'open_issues', transformer: numberToBigIntTransformer })
+    public openIssues!: number;
+
+    @Column({ name: 'network', transformer: numberToBigIntTransformer })
+    public network!: number;
+
+    @Column({ name: 'subscribers', transformer: numberToBigIntTransformer })
+    public subscribers!: number;
+
+}

packages/pipeline/src/entities/index.ts

@@ -8,6 +8,7 @@ export { EtherscanTransaction } from './etherscan_transaction';
 export { ExchangeCancelEvent } from './exchange_cancel_event';
 export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event';
 export { ExchangeFillEvent } from './exchange_fill_event';
+export { GithubRepo } from './github_repo';
 export { NonfungibleDotComTrade } from './nonfungible_dot_com_trade';
 export { OHLCVExternal } from './ohlcv_external';
 export { Relayer } from './relayer';

packages/pipeline/src/ormconfig.ts

@@ -13,6 +13,7 @@ import {
     ExchangeCancelEvent,
     ExchangeCancelUpToEvent,
     ExchangeFillEvent,
+    GithubRepo,
     NonfungibleDotComTrade,
     OHLCVExternal,
     Relayer,
@@ -37,6 +38,7 @@ const entities = [
     ExchangeCancelUpToEvent,
     ExchangeFillEvent,
     ERC20ApprovalEvent,
+    GithubRepo,
     NonfungibleDotComTrade,
     OHLCVExternal,
     Relayer,

packages/pipeline/src/parsers/github/index.ts

@@ -0,0 +1 @@
+export { parseGithubRepo } from './repo';

packages/pipeline/src/parsers/github/repo.ts

@@ -0,0 +1,23 @@
+import { GithubRepoResponse } from '../../data_sources/github';
+import { GithubRepo } from '../../entities';
+
+/**
+ * Converts a Github response from the API into an GithubRepo entity.
+ * @param rawRepo A Github response from the API into an GithubRepo entity.
+ */
+export function parseGithubRepo(rawRepo: GithubRepoResponse, observedTimestamp: number): GithubRepo {
+    const parsedRepo = new GithubRepo();
+    parsedRepo.observedTimestamp = observedTimestamp;
+    parsedRepo.name = rawRepo.name;
+    parsedRepo.createdAt = Date.parse(rawRepo.created_at);
+    parsedRepo.updatedAt = Date.parse(rawRepo.updated_at);
+    parsedRepo.pushedAt = Date.parse(rawRepo.pushed_at);
+    parsedRepo.size = rawRepo.size;
+    parsedRepo.stargazers = rawRepo.stargazers_count;
+    parsedRepo.watchers = rawRepo.watchers_count;
+    parsedRepo.forks = rawRepo.forks;
+    parsedRepo.openIssues = rawRepo.open_issues;
+    parsedRepo.network = rawRepo.network_count;
+    parsedRepo.subscribers = rawRepo.subscribers_count;
+    return parsedRepo;
+}

packages/pipeline/src/scripts/pull_github.ts

@@ -0,0 +1,28 @@
+import { Connection, ConnectionOptions, createConnection } from 'typeorm';
+
+import { GithubSource } from '../data_sources/github';
+import { GithubRepo } from '../entities';
+import * as ormConfig from '../ormconfig';
+import { parseGithubRepo } from '../parsers/github';
+import { handleError } from '../utils';
+
+const GITHUB_OWNER = '0xProject';
+const GITHUB_REPO = '0x-monorepo';
+
+let connection: Connection;
+
+(async () => {
+    connection = await createConnection(ormConfig as ConnectionOptions);
+    const GithubRepoRepository = connection.getRepository(GithubRepo);
+    const githubSource = new GithubSource(GITHUB_OWNER, GITHUB_REPO);
+
+    // get repo and save
+    const rawRepo = await githubSource.getGithubRepoAsync();
+    const observedTimestamp = Date.now();
+    const record = parseGithubRepo(rawRepo, observedTimestamp);
+    await GithubRepoRepository.save(record);
+
+    // TODO: get pull requests and save
+
+    process.exit(0);
+})().catch(handleError);

packages/pipeline/test/fixtures/github/api_v3_repo.json

@@ -0,0 +1,122 @@
+{
+    "id": 92181371,
+    "node_id": "MDEwOlJlcG9zaXRvcnk5MjE4MTM3MQ==",
+    "name": "0x-monorepo",
+    "full_name": "0xProject/0x-monorepo",
+    "private": false,
+    "owner": {
+      "login": "0xProject",
+      "id": 24832717,
+      "node_id": "MDEyOk9yZ2FuaXphdGlvbjI0ODMyNzE3",
+      "avatar_url": "https://avatars2.githubusercontent.com/u/24832717?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/0xProject",
+      "html_url": "https://github.com/0xProject",
+      "followers_url": "https://api.github.com/users/0xProject/followers",
+      "following_url": "https://api.github.com/users/0xProject/following{/other_user}",
+      "gists_url": "https://api.github.com/users/0xProject/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/0xProject/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/0xProject/subscriptions",
+      "organizations_url": "https://api.github.com/users/0xProject/orgs",
+      "repos_url": "https://api.github.com/users/0xProject/repos",
+      "events_url": "https://api.github.com/users/0xProject/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/0xProject/received_events",
+      "type": "Organization",
+      "site_admin": false
+    },
+    "html_url": "https://github.com/0xProject/0x-monorepo",
+    "description": "0x protocol monorepo - includes our smart contracts and many developer tools",
+    "fork": false,
+    "url": "https://api.github.com/repos/0xProject/0x-monorepo",
+    "forks_url": "https://api.github.com/repos/0xProject/0x-monorepo/forks",
+    "keys_url": "https://api.github.com/repos/0xProject/0x-monorepo/keys{/key_id}",
+    "collaborators_url": "https://api.github.com/repos/0xProject/0x-monorepo/collaborators{/collaborator}",
+    "teams_url": "https://api.github.com/repos/0xProject/0x-monorepo/teams",
+    "hooks_url": "https://api.github.com/repos/0xProject/0x-monorepo/hooks",
+    "issue_events_url": "https://api.github.com/repos/0xProject/0x-monorepo/issues/events{/number}",
+    "events_url": "https://api.github.com/repos/0xProject/0x-monorepo/events",
+    "assignees_url": "https://api.github.com/repos/0xProject/0x-monorepo/assignees{/user}",
+    "branches_url": "https://api.github.com/repos/0xProject/0x-monorepo/branches{/branch}",
+    "tags_url": "https://api.github.com/repos/0xProject/0x-monorepo/tags",
+    "blobs_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/blobs{/sha}",
+    "git_tags_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/tags{/sha}",
+    "git_refs_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/refs{/sha}",
+    "trees_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/trees{/sha}",
+    "statuses_url": "https://api.github.com/repos/0xProject/0x-monorepo/statuses/{sha}",
+    "languages_url": "https://api.github.com/repos/0xProject/0x-monorepo/languages",
+    "stargazers_url": "https://api.github.com/repos/0xProject/0x-monorepo/stargazers",
+    "contributors_url": "https://api.github.com/repos/0xProject/0x-monorepo/contributors",
+    "subscribers_url": "https://api.github.com/repos/0xProject/0x-monorepo/subscribers",
+    "subscription_url": "https://api.github.com/repos/0xProject/0x-monorepo/subscription",
+    "commits_url": "https://api.github.com/repos/0xProject/0x-monorepo/commits{/sha}",
+    "git_commits_url": "https://api.github.com/repos/0xProject/0x-monorepo/git/commits{/sha}",
+    "comments_url": "https://api.github.com/repos/0xProject/0x-monorepo/comments{/number}",
+    "issue_comment_url": "https://api.github.com/repos/0xProject/0x-monorepo/issues/comments{/number}",
+    "contents_url": "https://api.github.com/repos/0xProject/0x-monorepo/contents/{+path}",
+    "compare_url": "https://api.github.com/repos/0xProject/0x-monorepo/compare/{base}...{head}",
+    "merges_url": "https://api.github.com/repos/0xProject/0x-monorepo/merges",
+    "archive_url": "https://api.github.com/repos/0xProject/0x-monorepo/{archive_format}{/ref}",
+    "downloads_url": "https://api.github.com/repos/0xProject/0x-monorepo/downloads",
+    "issues_url": "https://api.github.com/repos/0xProject/0x-monorepo/issues{/number}",
+    "pulls_url": "https://api.github.com/repos/0xProject/0x-monorepo/pulls{/number}",
+    "milestones_url": "https://api.github.com/repos/0xProject/0x-monorepo/milestones{/number}",
+    "notifications_url": "https://api.github.com/repos/0xProject/0x-monorepo/notifications{?since,all,participating}",
+    "labels_url": "https://api.github.com/repos/0xProject/0x-monorepo/labels{/name}",
+    "releases_url": "https://api.github.com/repos/0xProject/0x-monorepo/releases{/id}",
+    "deployments_url": "https://api.github.com/repos/0xProject/0x-monorepo/deployments",
+    "created_at": "2017-05-23T14:17:33Z",
+    "updated_at": "2019-03-06T21:48:49Z",
+    "pushed_at": "2019-03-06T23:59:05Z",
+    "git_url": "git://github.com/0xProject/0x-monorepo.git",
+    "ssh_url": "git@github.com:0xProject/0x-monorepo.git",
+    "clone_url": "https://github.com/0xProject/0x-monorepo.git",
+    "svn_url": "https://github.com/0xProject/0x-monorepo",
+    "homepage": "",
+    "size": 86538,
+    "stargazers_count": 989,
+    "watchers_count": 989,
+    "language": "TypeScript",
+    "has_issues": true,
+    "has_projects": false,
+    "has_downloads": true,
+    "has_wiki": false,
+    "has_pages": false,
+    "forks_count": 294,
+    "mirror_url": null,
+    "archived": false,
+    "open_issues_count": 46,
+    "license": {
+      "key": "other",
+      "name": "Other",
+      "spdx_id": "NOASSERTION",
+      "url": null,
+      "node_id": "MDc6TGljZW5zZTA="
+    },
+    "forks": 294,
+    "open_issues": 46,
+    "watchers": 989,
+    "default_branch": "development",
+    "organization": {
+      "login": "0xProject",
+      "id": 24832717,
+      "node_id": "MDEyOk9yZ2FuaXphdGlvbjI0ODMyNzE3",
+      "avatar_url": "https://avatars2.githubusercontent.com/u/24832717?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/0xProject",
+      "html_url": "https://github.com/0xProject",
+      "followers_url": "https://api.github.com/users/0xProject/followers",
+      "following_url": "https://api.github.com/users/0xProject/following{/other_user}",
+      "gists_url": "https://api.github.com/users/0xProject/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/0xProject/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/0xProject/subscriptions",
+      "organizations_url": "https://api.github.com/users/0xProject/orgs",
+      "repos_url": "https://api.github.com/users/0xProject/repos",
+      "events_url": "https://api.github.com/users/0xProject/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/0xProject/received_events",
+      "type": "Organization",
+      "site_admin": false
+    },
+    "network_count": 294,
+    "subscribers_count": 89
+  }
+  

packages/pipeline/test/fixtures/github/api_v3_repo.ts

@@ -0,0 +1,21 @@
+import { GithubRepo } from '../../../src/entities';
+
+// To re-create the JSON file from the API (e.g. if the API output schema changes), run the below command:
+// curl https://api.github.com/repos/0xProject/0x-monorepo
+// docs here: https://developer.github.com/v3/repos/#get
+
+const ParsedGithubRepo: GithubRepo = {
+        observedTimestamp: Date.now(),
+        name: '0x-monorepo',
+        createdAt: 1495549053000,
+        updatedAt: 1551908929000,
+        pushedAt: 1551916745000,
+        size: 86538,
+        stargazers: 989,
+        watchers: 989,
+        forks: 294,
+        openIssues: 46,
+        network: 294,
+        subscribers: 89,
+      };
+export { ParsedGithubRepo };

packages/pipeline/test/parsers/github/github_repo_test.ts

@@ -0,0 +1,25 @@
+import * as chai from 'chai';
+import 'mocha';
+
+import { GithubRepoResponse } from '../../../src/data_sources/github';
+import { parseGithubRepo } from '../../../src/parsers/github';
+import { chaiSetup } from '../../utils/chai_setup';
+
+import { ParsedGithubRepo } from '../../fixtures/github/api_v3_repo';
+import * as githubRepoResponse from '../../fixtures/github/api_v3_repo.json';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('github_repo', () => {
+    describe('parseGithubRepo', () => {
+        it('converts GithubResponse to GithubRepo entities', () => {
+            const response: GithubRepoResponse = githubRepoResponse;
+            const expected = ParsedGithubRepo;
+            const observedTimestamp = expected.observedTimestamp;
+            const actual = parseGithubRepo(response, observedTimestamp);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});

packages/pipeline/tsconfig.json

@@ -14,6 +14,7 @@
         "./test/fixtures/copper/api_v1_list_activities.json",
         "./test/fixtures/copper/api_v1_list_leads.json",
         "./test/fixtures/copper/api_v1_list_opportunities.json",
-        "./test/fixtures/etherscan/api_v1_accounts_transactions.json"
+        "./test/fixtures/etherscan/api_v1_accounts_transactions.json",
+        "./test/fixtures/github/api_v3_repo.json"
     ]
 }