crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor all docJSON's to reside within the same S3 bucket under different folders

Browse Source
This commit is contained in:
Fabio Berger 2018-03-16 16:56:53 +01:00
parent 5a827eb3d4
commit d7bf003d51
8 changed files with 72 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/0x.js/package.json
View File

@ -36,8 +36,8 @@
"assets": ["_bundles/index.js", "_bundles/index.min.js"], "assets": ["_bundles/index.js", "_bundles/index.min.js"],
"docPublishConfigs": { "docPublishConfigs": {
"extraFileIncludes": ["../types/src/index.ts"], "extraFileIncludes": ["../types/src/index.ts"],
"s3BucketPath": "s3://0xjs-docs-jsons/", "s3BucketPath": "s3://doc-jsons/0xjs/",
"s3StagingBucketPath": "s3://staging-0xjs-docs-jsons/" "s3StagingBucketPath": "s3://staging-doc-jsons/0xjs/"
} }
} }
}, },

4
packages/connect/package.json
View File

@ -29,8 +29,8 @@
"postpublish": { "postpublish": {
"assets": ["_bundles/index.js", "_bundles/index.min.js"], "assets": ["_bundles/index.js", "_bundles/index.min.js"],
"docPublishConfigs": { "docPublishConfigs": {
"s3BucketPath": "s3://connect-docs-jsons/", "s3BucketPath": "s3://doc-jsons/connect/",
"s3StagingBucketPath": "s3://staging-connect-docs-jsons/" "s3StagingBucketPath": "s3://staging-doc-jsons/connect/"
} }
} }
}, },

4
packages/deployer/package.json
View File

@ -23,8 +23,8 @@
"assets": [], "assets": [],
"docPublishConfigs": { "docPublishConfigs": {
"extraFileIncludes": ["../types/src/index.ts"], "extraFileIncludes": ["../types/src/index.ts"],
"s3BucketPath": "s3://depoyer-docs-jsons/", "s3BucketPath": "s3://doc-jsons/deployer/",
"s3StagingBucketPath": "s3://staging-depoyer-docs-jsons/" "s3StagingBucketPath": "s3://staging-doc-jsons/deployer/"
} }
} }
}, },

4
packages/web3-wrapper/package.json
View File

@ -16,8 +16,8 @@
"postpublish": { "postpublish": {
"docPublishConfigs": { "docPublishConfigs": {
"extraFileIncludes": ["../types/src/index.ts"], "extraFileIncludes": ["../types/src/index.ts"],
"s3BucketPath": "s3://web3-wrapper-docs-json/", "s3BucketPath": "s3://doc-jsons/web3-wrapper/",
"s3StagingBucketPath": "s3://staging-web3-wrapper-docs-json/" "s3StagingBucketPath": "s3://staging-doc-jsons/web3-wrapper/"
} }
} }
}, },

28
packages/website/ts/pages/documentation/doc_page.tsx
View File

@ -17,13 +17,13 @@ import { utils } from 'ts/utils/utils';
const ZERO_EX_JS_VERSION_MISSING_TOPLEVEL_PATH = '0.32.4'; const ZERO_EX_JS_VERSION_MISSING_TOPLEVEL_PATH = '0.32.4';
const isDevelopment = configs.ENVIRONMENT === Environments.DEVELOPMENT; const isDevelopment = configs.ENVIRONMENT !== Environments.DEVELOPMENT;
const docIdToS3BucketName: { [id: string]: string } = { const docIdToS3FolderName: { [id: string]: string } = {
[DocPackages.ZeroExJs]: isDevelopment ? 'staging-0xjs-docs-jsons' : '0xjs-docs-jsons', [DocPackages.ZeroExJs]: '0xjs',
[DocPackages.SmartContracts]: 'smart-contracts-docs-json', [DocPackages.SmartContracts]: 'smart-contracts',
[DocPackages.Connect]: isDevelopment ? 'staging-connect-docs-jsons' : 'connect-docs-jsons', [DocPackages.Connect]: 'connect',
[DocPackages.Web3Wrapper]: isDevelopment ? 'staging-web3-wrapper-docs-json' : 'web3-wrapper-docs-json', [DocPackages.Web3Wrapper]: 'web3-wrapper',
[DocPackages.Deployer]: isDevelopment ? 'staging-depoyer-docs-jsons' : 'depoyer-docs-jsons', [DocPackages.Deployer]: 'deployer',
}; };
const docIdToSubpackageName: { [id: string]: string } = { const docIdToSubpackageName: { [id: string]: string } = {
@ -101,25 +101,25 @@ export class DocPage extends React.Component<DocPageProps, DocPageState> {
); );
} }
private async _fetchJSONDocsFireAndForgetAsync(preferredVersionIfExists?: string): Promise<void> { private async _fetchJSONDocsFireAndForgetAsync(preferredVersionIfExists?: string): Promise<void> {
const s3BucketName = docIdToS3BucketName[this.props.docsInfo.id]; const folderName = docIdToS3FolderName[this.props.docsInfo.id];
const docsJsonRoot = `${constants.S3_BUCKET_ROOT}/${s3BucketName}`; const docBucketRoot = isDevelopment ? constants.S3_STAGING_DOC_BUCKET_ROOT : constants.S3_DOC_BUCKET_ROOT;
const versionToFileName = await docUtils.getVersionToFileNameAsync(docsJsonRoot); const versionToFilePath = await docUtils.getVersionToFilePathAsync(docBucketRoot, folderName);
const versions = _.keys(versionToFileName); const versions = _.keys(versionToFilePath);
this.props.dispatcher.updateAvailableDocVersions(versions); this.props.dispatcher.updateAvailableDocVersions(versions);
const sortedVersions = semverSort.desc(versions); const sortedVersions = semverSort.desc(versions);
const latestVersion = sortedVersions[0]; const latestVersion = sortedVersions[0];
let versionToFetch = latestVersion; let versionToFetch = latestVersion;
if (!_.isUndefined(preferredVersionIfExists)) { if (!_.isUndefined(preferredVersionIfExists)) {
const preferredVersionFileNameIfExists = versionToFileName[preferredVersionIfExists]; const preferredVersionFileNameIfExists = versionToFilePath[preferredVersionIfExists];
if (!_.isUndefined(preferredVersionFileNameIfExists)) { if (!_.isUndefined(preferredVersionFileNameIfExists)) {
versionToFetch = preferredVersionIfExists; versionToFetch = preferredVersionIfExists;
} }
} }
this.props.dispatcher.updateCurrentDocsVersion(versionToFetch); this.props.dispatcher.updateCurrentDocsVersion(versionToFetch);
const versionFileNameToFetch = versionToFileName[versionToFetch]; const versionFilePathToFetch = versionToFilePath[versionToFetch];
const versionDocObj = await docUtils.getJSONDocFileAsync(versionFileNameToFetch, docsJsonRoot); const versionDocObj = await docUtils.getJSONDocFileAsync(versionFilePathToFetch, docBucketRoot);
const docAgnosticFormat = this.props.docsInfo.convertToDocAgnosticFormat(versionDocObj); const docAgnosticFormat = this.props.docsInfo.convertToDocAgnosticFormat(versionDocObj);
if (!this._isUnmounted) { if (!this._isUnmounted) {

2
packages/website/ts/types.ts
View File

@ -336,7 +336,7 @@ export enum TokenVisibility {
TRACKED = 'TRACKED', TRACKED = 'TRACKED',
} }
export interface VersionToFileName { export interface VersionToFilePath {
[version: string]: string; [version: string]: string;
} }

3
packages/website/ts/utils/constants.ts
View File

@ -29,7 +29,8 @@ export const constants = {
PROVIDER_NAME_GENERIC: 'Injected Web3', PROVIDER_NAME_GENERIC: 'Injected Web3',
PROVIDER_NAME_PUBLIC: '0x Public', PROVIDER_NAME_PUBLIC: '0x Public',
ROLLBAR_ACCESS_TOKEN: 'a6619002b51c4464928201e6ea94de65', ROLLBAR_ACCESS_TOKEN: 'a6619002b51c4464928201e6ea94de65',
S3_BUCKET_ROOT: 'https://s3.amazonaws.com', S3_DOC_BUCKET_ROOT: 'https://s3.amazonaws.com/doc-jsons',
S3_STAGING_DOC_BUCKET_ROOT: 'https://s3.amazonaws.com/staging-doc-jsons',
SUCCESS_STATUS: 200, SUCCESS_STATUS: 200,
UNAVAILABLE_STATUS: 503, UNAVAILABLE_STATUS: 503,
TAKER_FEE: new BigNumber(0), TAKER_FEE: new BigNumber(0),

60
packages/website/ts/utils/doc_utils.ts
View File

@ -2,21 +2,21 @@ import { DoxityDocObj, TypeDocNode } from '@0xproject/react-docs';
import { logUtils } from '@0xproject/utils'; import { logUtils } from '@0xproject/utils';
import findVersions = require('find-versions'); import findVersions = require('find-versions');
import * as _ from 'lodash'; import * as _ from 'lodash';
import { S3FileObject, VersionToFileName } from 'ts/types'; import { S3FileObject, VersionToFilePath } from 'ts/types';
import { utils } from 'ts/utils/utils'; import { utils } from 'ts/utils/utils';
import convert = require('xml-js'); import convert = require('xml-js');
export const docUtils = { export const docUtils = {
async getVersionToFileNameAsync(s3DocJsonRoot: string): Promise<VersionToFileName> { async getVersionToFilePathAsync(s3DocJsonRoot: string, folderName: string): Promise<VersionToFilePath> {
const versionFileNames = await this.getVersionFileNamesAsync(s3DocJsonRoot); const versionFilePaths = await this.getVersionFileNamesAsync(s3DocJsonRoot, folderName);
const versionToFileName: VersionToFileName = {}; const versionToFilePath: VersionToFilePath = {};
_.each(versionFileNames, fileName => { _.each(versionFilePaths, filePath => {
const [version] = findVersions(fileName); const [version] = findVersions(filePath);
versionToFileName[version] = fileName; versionToFilePath[version] = filePath;
}); });
return versionToFileName; return versionToFilePath;
}, },
async getVersionFileNamesAsync(s3DocJsonRoot: string): Promise<string[]> { async getVersionFileNamesAsync(s3DocJsonRoot: string, folderName: string): Promise<string[]> {
const response = await fetch(s3DocJsonRoot); const response = await fetch(s3DocJsonRoot);
if (response.status !== 200) { if (response.status !== 200) {
// TODO: Show the user an error message when the docs fail to load // TODO: Show the user an error message when the docs fail to load
@ -33,13 +33,47 @@ export const docUtils = {
? (responseObj.ListBucketResult.Contents as S3FileObject[]) ? (responseObj.ListBucketResult.Contents as S3FileObject[])
: [responseObj.ListBucketResult.Contents]; : [responseObj.ListBucketResult.Contents];
const versionFileNames = _.map(fileObjs, fileObj => { /*
* S3 simply pre-fixes files in "folders" with the folder name. Thus, since we
* store docJSONs for multiple packages in a single S3 bucket, we must filter out
* the versionFileNames for a given folder here (ignoring folder entries)
*
* Example S3 response:
* <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
* <Name>staging-doc-jsons</Name>
* <Prefix/>
* <Marker/>
* <MaxKeys>1000</MaxKeys>
* <IsTruncated>false</IsTruncated>
* <Contents>
* <Key>0xjs/</Key>
* <LastModified>2018-03-16T13:17:46.000Z</LastModified>
* <ETag>"d41d8cd98f00b204e9800998ecf8427e"</ETag>
* <Size>0</Size>
* <StorageClass>STANDARD</StorageClass>
* </Contents>
* <Contents>
* <Key>0xjs/v0.1.0.json</Key>
* <LastModified>2018-03-16T13:18:23.000Z</LastModified>
* <ETag>"b4f7f74913aab4a5ad1e6a58fcb3b274"</ETag>
* <Size>1039050</Size>
* <StorageClass>STANDARD</StorageClass>
* </Contents>
*/
const relevantObjs = _.filter(fileObjs, fileObj => {
const key = fileObj.Key._text;
const isInFolderOfInterest = _.includes(key, folderName);
const isFileEntry = !_.endsWith(key, '/');
return isInFolderOfInterest && isFileEntry;
});
const versionFilePaths = _.map(relevantObjs, fileObj => {
return fileObj.Key._text; return fileObj.Key._text;
}); });
return versionFileNames; return versionFilePaths;
}, },
async getJSONDocFileAsync(fileName: string, s3DocJsonRoot: string): Promise<TypeDocNode | DoxityDocObj> { async getJSONDocFileAsync(filePath: string, s3DocJsonRoot: string): Promise<TypeDocNode | DoxityDocObj> {
const endpoint = `${s3DocJsonRoot}/${fileName}`; const endpoint = `${s3DocJsonRoot}/${filePath}`;
const response = await fetch(endpoint); const response = await fetch(endpoint);
if (response.status !== 200) { if (response.status !== 200) {
// TODO: Show the user an error message when the docs fail to load // TODO: Show the user an error message when the docs fail to load