crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Throttling to respect rate limits

Browse Source
This commit is contained in:
askeluv 2019-03-22 18:15:49 +08:00
parent 346c6fc590
commit 08e49dcf2e
2 changed files with 27 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
packages/pipeline/src/data_sources/github/index.ts
View File

@ -1,4 +1,7 @@
import { fetchAsync, logUtils } from '@0x/utils';
import Bottleneck from 'bottleneck';
const ONE_SECOND = 1000;
export interface GithubRepoResponse {
full_name: string;
@ -68,17 +71,22 @@ export interface GithubIssueResponse {
// tslint:disable:prefer-function-over-method
// ^ Keep consistency with other sources and help logical organization
export class GithubSource {
public readonly _urlBase: string = 'https://api.github.com';
public readonly _owner: string;
public readonly _repo: string;
public readonly _branch: string;
public readonly _accessToken: string;
// rate-limit for all API calls through this class instance
private readonly _limiter: Bottleneck;
private readonly _urlBase: string = 'https://api.github.com';
private readonly _owner: string;
private readonly _repo: string;
private readonly _branch: string;
private readonly _accessToken: string;
constructor(owner: string, repo: string, branch: string, accessToken: string) {
constructor(owner: string, repo: string, branch: string, accessToken: string, maxReqsPerSecond: number) {
this._owner = owner;
this._repo = repo;
this._branch = branch;
this._accessToken = accessToken;
this._limiter = new Bottleneck({
minTime: ONE_SECOND / maxReqsPerSecond,
});
}
/**
@ -86,7 +94,7 @@ export class GithubSource {
*/
public async getGithubRepoAsync(): Promise<GithubRepoResponse> {
const url = `${this._urlBase}/repos/${this._owner}/${this._repo}?access_token=${this._accessToken}`;
const resp = await fetchAsync(url);
const resp = await this._limiter.schedule(() => fetchAsync(url));
const respJson: GithubRepoResponse = await resp.json();
return respJson;
}
@ -98,7 +106,7 @@ export class GithubSource {
const url = `${this._urlBase}/repos/${this._owner}/${this._repo}/pulls?access_token=${
this._accessToken
}&state=all&per_page=100&page=${page}`;
const resp = await fetchAsync(url);
const resp = await this._limiter.schedule(() => fetchAsync(url));
const respJson: GithubPullRequestResponse[] = await resp.json();
return respJson;
}
@ -110,7 +118,7 @@ export class GithubSource {
const url = `${this._urlBase}/repos/${this._owner}/${this._repo}/forks?access_token=${
this._accessToken
}&per_page=100&page=${page}`;
const resp = await fetchAsync(url);
const resp = await this._limiter.schedule(() => fetchAsync(url));
const respJson: GithubForkResponse[] = await resp.json();
return respJson;
}
@ -122,7 +130,7 @@ export class GithubSource {
const url = `${this._urlBase}/repos/${this._owner}/${this._repo}/compare/${
this._branch
}...${forkOwner}:${forkBranch}?access_token=${this._accessToken}`;
const resp = await fetchAsync(url);
const resp = await this._limiter.schedule(() => fetchAsync(url));
const respJson: GithubComparisonResponse = await resp.json();
return respJson;
}
@ -134,7 +142,7 @@ export class GithubSource {
const url = `${this._urlBase}/repos/${this._owner}/${this._repo}/issues?access_token=${
this._accessToken
}&state=all&per_page=100&page=${page}`;
const resp = await fetchAsync(url);
const resp = await this._limiter.schedule(() => fetchAsync(url));
const respJson: GithubIssueResponse[] = await resp.json();
if (!respJson[0]) {
logUtils.log(respJson);

9
packages/pipeline/src/scripts/pull_github.ts
View File

@ -18,6 +18,7 @@ const GITHUB_OWNER = '0xProject';
const GITHUB_REPO = '0x-monorepo';
const GITHUB_BRANCH = 'development';
const RECORDS_PER_PAGE = 100;
const MAX_REQUESTS_PER_SECOND = 1;
let connection: Connection;
@ -27,7 +28,13 @@ let connection: Connection;
if (accessToken === undefined) {
throw new Error('Missing required env var: GITHUB_ACCESS_TOKEN');
}
const githubSource = new GithubSource(GITHUB_OWNER, GITHUB_REPO, GITHUB_BRANCH, accessToken);
const githubSource = new GithubSource(
GITHUB_OWNER,
GITHUB_REPO,
GITHUB_BRANCH,
accessToken,
MAX_REQUESTS_PER_SECOND,
);
const observedTimestamp = Date.now();
// get repo and save