crowetic/commerce
1
0
Fork 1
mirror of https://github.com/vercel/commerce.git synced 2025-06-08 17:26:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

tmp file management

Browse Source
This commit is contained in:
Joel Varty 2021-01-18 12:39:16 -05:00
parent c7a8dbdf87
commit d15dc67414
7 changed files with 323 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
framework/agility/agility.config.js
View File

@ -1,54 +1,74 @@
const agilityContentSync = require("@agility/content-sync");
const agilityFileSystem = require("@agility/content-sync/src/store-interface-filesystem");
const nextFileSystem = require("./next.file.sync");
const path = require("path")
const fs = require('fs-extra');
const agilityConfig = {
guid: process.env.AGILITY_GUID, //Set your guid here
fetchAPIKey: process.env.AGILITY_API_FETCH_KEY, //Set your fetch apikey here
previewAPIKey: process.env.AGILITY_API_PREVIEW_KEY, //set your preview apikey
languageCode: "en-us", //the language for your website in Agility CMS
channelName: "website", //the name of your channel in Agility CMS
securityKey: process.env.AGILITY_SECURITY_KEY, //the website security key used to validate and generate preview keys
guid: process.env.AGILITY_GUID, //Set your guid here
fetchAPIKey: process.env.AGILITY_API_FETCH_KEY, //Set your fetch apikey here
previewAPIKey: process.env.AGILITY_API_PREVIEW_KEY, //set your preview apikey
languageCode: "en-us", //the language for your website in Agility CMS
channelName: "website", //the name of your channel in Agility CMS
securityKey: process.env.AGILITY_SECURITY_KEY, //the website security key used to validate and generate preview keys
};
const getSyncClient = ({ isPreview, isDevelopmentMode }) => {
const getSyncClient = ({ isPreview, isDevelopmentMode, isIncremental }) => {
const rootPath = process.cwd()
let cachePath = `${rootPath}/.next/cache/agility/${
isPreview ? "preview" : "live"
}`;
let cachePath = `${rootPath}/.next/cache/agility/${isPreview ? "preview" : "live" }`;
// if (!isDevelopmentMode) {
// //we are in prod and need to use a different directory that Vercel understands
// cachePath = `/tmp/agilitycache/${isPreview ? "preview" : "live"}`;
// }
//if we are in "incremental" mode, we need to use the tmp folder...
if (isIncremental) {
cachePath = `/tmp/agilitycache/${isPreview ? "preview" : "live"}`;
}
console.log(`Agility CMS => Content cache path is ${cachePath}`);
const apiKey = isPreview
? agilityConfig.previewAPIKey
: agilityConfig.fetchAPIKey;
console.log(`AgilityCMS => Content cache path is ${cachePath}`);
const apiKey = isPreview
? agilityConfig.previewAPIKey
: agilityConfig.fetchAPIKey;
if (!agilityConfig.guid) {
console.log("Agility CMS => No GUID was provided.");
return null;
}
if (!agilityConfig.guid) {
console.log("AgilityCMS => No GUID was provided.");
return null;
}
return agilityContentSync.getSyncClient({
guid: agilityConfig.guid,
apiKey: apiKey,
isPreview: isPreview,
languages: [agilityConfig.languageCode],
channels: [agilityConfig.channelName],
store: {
interface: agilityFileSystem,
options: {
rootPath: cachePath,
},
},
});
return agilityContentSync.getSyncClient({
guid: agilityConfig.guid,
apiKey: apiKey,
isPreview: isPreview,
languages: [agilityConfig.languageCode],
channels: [agilityConfig.channelName],
store: {
interface: nextFileSystem,
options: {
rootPath: cachePath
},
},
});
};
const prepIncrementalMode = async () => {
let cachePath = `${rootPath}/.next/cache/agility/`;
const tempPath = `/tmp/agilitycache/`;
const buildFilePath = path.join(tempPath, "build.log")
//check for the build file in here...
if (!fs.existsSync(buildFilePath)) {
console.log(`Copying Agility Content files to temp folder: ${tempPath}`)
//copy everything across from cachePath
await fs.copy(cachePath, tempPath)
}
}
module.exports = {
agilityConfig,
getSyncClient,
agilityConfig,
getSyncClient,
prepIncrementalMode
};

51
framework/agility/agility.node.ts
View File

@ -1,10 +1,11 @@
import fs from "fs"
import crypto from 'crypto'
import { asyncForEach } from "./utils"
import { ModuleWithInit } from "@agility/types"
//Agility API stuff
import { agilityConfig, getSyncClient } from './agility.config'
import { agilityConfig, getSyncClient, prepIncrementalMode } from './agility.config'
import GlobalFooter from 'components/agility-global/GlobalFooter'
import GlobalHeader from 'components/agility-global/GlobalHeader'
@ -41,35 +42,37 @@ const getAgilityPageProps = async ({ params, preview, locale }:any):Promise<Agil
})
}
//determine if we've already done a full build yet
const buildFilePath = `${process.cwd()}/.next/cache/agility/build.log`
const isBuildComplete = fs.existsSync(buildFilePath)
//TODO: use locale to determin LANGUAGECODE (pulled from config at this point...)
//determine if we are in preview mode
const isPreview:boolean = (preview || isDevelopmentMode);
const agilitySyncClient = getSyncClient({
isPreview: isPreview,
isDevelopmentMode
isDevelopmentMode,
isIncremental: isBuildComplete
});
//always sync to get latest
if (! agilitySyncClient) {
console.log("Agility CMS => Sync client could not be accessed.")
console.log("AgilityCMS => Sync client could not be accessed.")
return {
notFound: true
};
}
if (!isDevelopmentMode) {
console.log(`Agility CMS => Syncing ${isPreview ? "Preview" : "Live"} Mode`)
if (preview || isBuildComplete) {
//only do on-demand sync in next's preview mode or incremental build...
console.log(`AgilityCMS => Sync On-demand ${isPreview ? "Preview" : "Live"} Mode`)
await prepIncrementalMode()
await agilitySyncClient.runSync();
}
console.log(`Agility CMS => Getting page props for '${path}'...`);
console.log(`AgilityCMS => Getting page props for '${path}'...`);
//get sitemap
const sitemap = await agilitySyncClient.store.getSitemap({ channelName, languageCode });
@ -135,7 +138,7 @@ const getAgilityPageProps = async ({ params, preview, locale }:any):Promise<Agil
//we have some additional data in the module we'll need, execute that method now, so it can be included in SSG
if (isDevelopmentMode) {
console.log(`Agility CMS => Fetching additional data via getCustomInitialProps for ${moduleItem.module}...`);
console.log(`AgilityCMS => Fetching additional data via getCustomInitialProps for ${moduleItem.module}...`);
}
const moduleData = await moduleInitializer({
@ -183,28 +186,28 @@ const getAgilityPageProps = async ({ params, preview, locale }:any):Promise<Agil
}
}
const getAgilityPaths = async () => {
const getAgilityPaths = async (preview:boolean|undefined) => {
console.log(`Agility CMS => Fetching sitemap for getAgilityPaths...`);
console.log(`AgilityCMS => Fetching sitemap for getAgilityPaths...`);
//determine if we are in preview mode
const isPreview = isDevelopmentMode;
const isPreview = isDevelopmentMode || preview;
//determine if we've already done a full build yet
const buildFilePath = `${process.cwd()}/.next/cache/agility/build.log`
const isBuildComplete = fs.existsSync(buildFilePath)
const agilitySyncClient = getSyncClient({
isPreview,
isDevelopmentMode
isDevelopmentMode,
isIncremental: isBuildComplete
});
//always sync to get latest
if (! agilitySyncClient) {
console.log("Agility CMS => Sync client could not be accessed.")
console.log("AgilityCMS => Sync client could not be accessed.")
return [];
}
if (!isDevelopmentMode) {
console.log(`Agility CMS => Syncing ${isPreview ? "Preview" : "Live"} Mode`)
await agilitySyncClient.runSync();
}
const sitemapFlat = await agilitySyncClient.store.getSitemap({
channelName,
@ -214,7 +217,7 @@ const getAgilityPaths = async () => {
if (!sitemapFlat) {
console.warn("Agility CMS => No Site map found. Make sure your environment variables are setup correctly.")
console.warn("AgilityCMS => No Site map found. Make sure your environment variables are setup correctly.")
return []
}

57
framework/agility/agility.sync.js
View File

@ -1,8 +1,5 @@
/*
THIS FILE IS ONLY EXECUTED LOCALLY
WHEN DOING A LOCAL SYNC ON DEMAND
IN DEVELOPMENT MODE
*/
const fs = require('fs')
require("dotenv").config({
path: `.env.local`,
@ -12,22 +9,44 @@ const { getSyncClient } = require('./agility.config')
const runSync = async () => {
setBuildLog(false)
const agilitySyncClient = getSyncClient({ isPreview: true, isDevelopmentMode: true })
if (! agilitySyncClient) {
console.log("Agility CMS => Sync client could not be accessed.")
console.log("AgilityCMS => Sync client could not be accessed.")
return;
}
await agilitySyncClient.runSync();
}
const syncAll = async () => {
const setBuildLog = (builtYN) => {
//clear out a file saying WE HAVE SYNC'D
const rootPath = process.cwd()
const filePath = `${rootPath}/.next/cache/agility/build.log`
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
}
if (builtYN) {
//write out the build log so we know that we are up to date
fs.writeFileSync(filePath, "BUILT");
} else {
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
}
}
}
const preBuild = async () => {
//clear the build log
setBuildLog(false)
//sync preview mode
let agilitySyncClient = getSyncClient({ isPreview: true, isDevelopmentMode: false })
if (! agilitySyncClient) {
console.log("Agility CMS => Sync client could not be accessed.")
console.log("AgilityCMS => Sync client could not be accessed.")
return;
}
@ -36,18 +55,27 @@ const syncAll = async () => {
//sync production mode
agilitySyncClient = getSyncClient({ isPreview: false, isDevelopmentMode: false })
if (! agilitySyncClient) {
console.log("Agility CMS => Sync client could not be accessed.")
console.log("AgilityCMS => Sync client could not be accessed.")
return;
}
await agilitySyncClient.runSync();
}
const postBuild = async() => {
//mark the build log as BUILT
setBuildLog(true)
}
const clearSync = async () => {
setBuildLog(false)
const agilitySyncClient = getSyncClient({ isPreview: true, isDevelopmentMode: true })
if (! agilitySyncClient) {
console.log("Agility CMS => Sync client could not be accessed.")
console.log("AgilityCMS => Sync client could not be accessed.")
return;
}
await agilitySyncClient.clearSync();
@ -63,10 +91,13 @@ if (process.argv[2]) {
//run the sync
return runSync()
} else if (process.argv[2] === "sync-all") {
//sync both staging and live content
return syncAll()
} else if (process.argv[2] === "prebuild") {
//pre build actions
return preBuild()
} else if (process.argv[2] === "postbuild") {
//post build actions
return postBuild()
}
}

153
framework/agility/next.file.sync.js Normal file
View File

@ -0,0 +1,153 @@
const fs = require('fs')
const os = require('os')
const path = require('path')
const { lockSync, unlockSync, checkSync, check } = require("proper-lockfile")
require("dotenv").config({
path: `.env.${process.env.NODE_ENV}`,
})
const sleep = (ms) => {
return new Promise(resolve => setTimeout(resolve, ms));
}
const getFilePath = ({ options, itemType, languageCode, itemID }) => {
const fileName = `${itemID}.json`;
return path.join(options.rootPath, languageCode, itemType, fileName);
}
const saveItem = async ({ options, item, itemType, languageCode, itemID }) => {
let filePath = getFilePath({ options, itemType, languageCode, itemID });
let dirPath = path.dirname(filePath);
if (!fs.existsSync(dirPath)) {
fs.mkdirSync(dirPath, { recursive: true });
}
let json = JSON.stringify(item);
fs.writeFileSync(filePath, json);
}
const deleteItem = async ({ options, itemType, languageCode, itemID }) => {
let filePath = getFilePath({ options, itemType, languageCode, itemID });
if (fs.existsSync(filePath)) {
fs.unlinkSync(filePath);
}
}
const mergeItemToList = async ({ options, item, languageCode, itemID, referenceName, definitionName }) => {
let contentList = await getItem({ options, itemType: "list", languageCode, itemID: referenceName });
if (contentList == null) {
//initialize the list
contentList = [item];
} else {
//replace the item...
const cIndex = contentList.findIndex((ci) => {
return ci.contentID === itemID;
});
if (item.properties.state === 3) {
//*** deleted item (remove from the list) ***
if (cIndex >= 0) {
//remove the item
contentList.splice(cIndex, 1);
}
} else {
//*** regular item (merge) ***
if (cIndex >= 0) {
//replace the existing item
contentList[cIndex] = item;
} else {
//and it to the end of the
contentList.push(item);
}
}
}
await saveItem({ options, item: contentList, itemType: "list", languageCode, itemID: referenceName });
}
const getItem = async ({ options, itemType, languageCode, itemID }) => {
let filePath = getFilePath({ options, itemType, languageCode, itemID });
if (!fs.existsSync(filePath)) return null;
let json = fs.readFileSync(filePath, 'utf8');
return JSON.parse(json);
}
const clearItems = async ({ options }) => {
fs.rmdirSync(options.rootPath, { recursive: true })
}
const waitOnLock = async (lockFile) => {
while (await check(lockFile)) {
await sleep(100)
}
}
const mutexLock = async () => {
const dir = os.tmpdir();
const lockFile = `${dir}/${"agility-sync"}.mutex`
if (! fs.existsSync(lockFile)) {
fs.writeFileSync(lockFile, "agility-sync");
}
//THE LOCK IS ALREADY HELD - WAIT UP!
await waitOnLock(lockFile)
try {
return lockSync(lockFile)
} catch (err) {
if (`${err}`.indexOf("Lock file is already being held") !== -1) {
//this error happens when 2 processes try to get a lock at the EXACT same time (very rare)
await sleep(100)
await waitOnLock(lockFile)
try {
return lockSync(lockFile)
} catch (e2) {
if (`${err}`.indexOf("Lock file is already being held") !== -1) {
//this error happens when 2 processes try to get a lock at the EXACT same time (very rare)
await sleep(100)
await waitOnLock(lockFile)
return lockSync(lockFile)
}
}
}
throw Error("The mutex lock could not be obtained.")
}
}
module.exports = {
saveItem,
deleteItem,
mergeItemToList,
getItem,
clearItems,
mutexLock
}

7
package.json
View File

@ -2,9 +2,11 @@
"name": "nextjs-commerce",
"version": "1.0.0",
"scripts": {
"dev": "next dev",
"prebuild": "node framework/agility/agility.sync.js sync-all",
"predev": "node framework/agility/agility.sync.js sync",
"dev": "next dev",
"prebuild": "node framework/agility/agility.sync.js prebuild",
"build": "next build",
"postbuild": "node framework/agility/agility.sync.js postbuild",
"start": "next start",
"analyze": "BUNDLE_ANALYZE=both yarn build",
"find:unused": "next-unused",
@ -56,6 +58,7 @@
"classnames": "^2.2.6",
"cookie": "^0.4.1",
"email-validator": "^2.0.4",
"fs-extra": "^9.0.1",
"js-cookie": "^2.2.1",
"keen-slider": "^5.2.4",
"lodash.debounce": "^4.0.8",

4
pages/[...slug].tsx
View File

@ -52,7 +52,7 @@ export async function getStaticProps({ preview, params, locale }: GetStaticProps
}
}
const pages = await getAgilityPaths()
const pages = await getAgilityPaths(preview)
if (!page) {
// We throw to make sure this fails at build time as this is never expected to happen
@ -68,7 +68,7 @@ export async function getStaticProps({ preview, params, locale }: GetStaticProps
export async function getStaticPaths({ locales }: GetStaticPathsContext) {
//get the paths configured in agility
let agilityPaths = await getAgilityPaths()
let agilityPaths = await getAgilityPaths(false)
//remove product/product-details from the agility paths (special details page...)
agilityPaths = agilityPaths.filter(p => p !== "/product/product-details")

34
yarn.lock
View File

@ -1750,6 +1750,11 @@ asynckit@^0.4.0:
resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
integrity sha1-x57Zf380y48robyXkLzDZkdLS3k=
at-least-node@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2"
integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==
atob@^2.1.2:
version "2.1.2"
resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9"
@ -3541,6 +3546,16 @@ fs-extra@^8.0.0:
jsonfile "^4.0.0"
universalify "^0.1.0"
fs-extra@^9.0.1:
version "9.0.1"
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.0.1.tgz#910da0062437ba4c39fedd863f1675ccfefcb9fc"
integrity sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==
dependencies:
at-least-node "^1.0.0"
graceful-fs "^4.2.0"
jsonfile "^6.0.1"
universalify "^1.0.0"
fs.realpath@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
@ -4391,6 +4406,15 @@ jsonfile@^4.0.0:
optionalDependencies:
graceful-fs "^4.1.6"
jsonfile@^6.0.1:
version "6.1.0"
resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae"
integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==
dependencies:
universalify "^2.0.0"
optionalDependencies:
graceful-fs "^4.1.6"
jsonify@~0.0.0:
version "0.0.0"
resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73"
@ -7423,6 +7447,16 @@ universalify@^0.1.0:
resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==
universalify@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/universalify/-/universalify-1.0.0.tgz#b61a1da173e8435b2fe3c67d29b9adf8594bd16d"
integrity sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==
universalify@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717"
integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==
unixify@1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/unixify/-/unixify-1.0.0.tgz#3a641c8c2ffbce4da683a5c70f03a462940c2090"