diff --git a/framework/agility/agility.config.js b/framework/agility/agility.config.js index 4ea96118e..02bc36e28 100644 --- a/framework/agility/agility.config.js +++ b/framework/agility/agility.config.js @@ -1,54 +1,74 @@ const agilityContentSync = require("@agility/content-sync"); -const agilityFileSystem = require("@agility/content-sync/src/store-interface-filesystem"); +const nextFileSystem = require("./next.file.sync"); + +const path = require("path") +const fs = require('fs-extra'); const agilityConfig = { - guid: process.env.AGILITY_GUID, //Set your guid here - fetchAPIKey: process.env.AGILITY_API_FETCH_KEY, //Set your fetch apikey here - previewAPIKey: process.env.AGILITY_API_PREVIEW_KEY, //set your preview apikey - languageCode: "en-us", //the language for your website in Agility CMS - channelName: "website", //the name of your channel in Agility CMS - securityKey: process.env.AGILITY_SECURITY_KEY, //the website security key used to validate and generate preview keys + guid: process.env.AGILITY_GUID, //Set your guid here + fetchAPIKey: process.env.AGILITY_API_FETCH_KEY, //Set your fetch apikey here + previewAPIKey: process.env.AGILITY_API_PREVIEW_KEY, //set your preview apikey + languageCode: "en-us", //the language for your website in Agility CMS + channelName: "website", //the name of your channel in Agility CMS + securityKey: process.env.AGILITY_SECURITY_KEY, //the website security key used to validate and generate preview keys }; -const getSyncClient = ({ isPreview, isDevelopmentMode }) => { +const getSyncClient = ({ isPreview, isDevelopmentMode, isIncremental }) => { const rootPath = process.cwd() - let cachePath = `${rootPath}/.next/cache/agility/${ - isPreview ? "preview" : "live" - }`; + let cachePath = `${rootPath}/.next/cache/agility/${isPreview ? "preview" : "live" }`; -// if (!isDevelopmentMode) { -// //we are in prod and need to use a different directory that Vercel understands -// cachePath = `/tmp/agilitycache/${isPreview ? "preview" : "live"}`; -// } + //if we are in "incremental" mode, we need to use the tmp folder... + if (isIncremental) { + cachePath = `/tmp/agilitycache/${isPreview ? "preview" : "live"}`; + } - console.log(`Agility CMS => Content cache path is ${cachePath}`); - const apiKey = isPreview - ? agilityConfig.previewAPIKey - : agilityConfig.fetchAPIKey; + console.log(`AgilityCMS => Content cache path is ${cachePath}`); + const apiKey = isPreview + ? agilityConfig.previewAPIKey + : agilityConfig.fetchAPIKey; - if (!agilityConfig.guid) { - console.log("Agility CMS => No GUID was provided."); - return null; - } + if (!agilityConfig.guid) { + console.log("AgilityCMS => No GUID was provided."); + return null; + } - return agilityContentSync.getSyncClient({ - guid: agilityConfig.guid, - apiKey: apiKey, - isPreview: isPreview, - languages: [agilityConfig.languageCode], - channels: [agilityConfig.channelName], - store: { - interface: agilityFileSystem, - options: { - rootPath: cachePath, - }, - }, - }); + return agilityContentSync.getSyncClient({ + guid: agilityConfig.guid, + apiKey: apiKey, + isPreview: isPreview, + languages: [agilityConfig.languageCode], + channels: [agilityConfig.channelName], + store: { + interface: nextFileSystem, + options: { + rootPath: cachePath + }, + }, + }); }; + +const prepIncrementalMode = async () => { + + let cachePath = `${rootPath}/.next/cache/agility/`; + const tempPath = `/tmp/agilitycache/`; + + const buildFilePath = path.join(tempPath, "build.log") + + //check for the build file in here... + if (!fs.existsSync(buildFilePath)) { + console.log(`Copying Agility Content files to temp folder: ${tempPath}`) + //copy everything across from cachePath + await fs.copy(cachePath, tempPath) + } +} + + + module.exports = { - agilityConfig, - getSyncClient, + agilityConfig, + getSyncClient, + prepIncrementalMode }; diff --git a/framework/agility/agility.node.ts b/framework/agility/agility.node.ts index 62d29c0ab..38bf3d8b5 100644 --- a/framework/agility/agility.node.ts +++ b/framework/agility/agility.node.ts @@ -1,10 +1,11 @@ +import fs from "fs" import crypto from 'crypto' import { asyncForEach } from "./utils" import { ModuleWithInit } from "@agility/types" //Agility API stuff -import { agilityConfig, getSyncClient } from './agility.config' +import { agilityConfig, getSyncClient, prepIncrementalMode } from './agility.config' import GlobalFooter from 'components/agility-global/GlobalFooter' import GlobalHeader from 'components/agility-global/GlobalHeader' @@ -41,35 +42,37 @@ const getAgilityPageProps = async ({ params, preview, locale }:any):Promise Sync client could not be accessed.") + console.log("AgilityCMS => Sync client could not be accessed.") return { notFound: true }; } - if (!isDevelopmentMode) { - console.log(`Agility CMS => Syncing ${isPreview ? "Preview" : "Live"} Mode`) + if (preview || isBuildComplete) { + //only do on-demand sync in next's preview mode or incremental build... + console.log(`AgilityCMS => Sync On-demand ${isPreview ? "Preview" : "Live"} Mode`) + + await prepIncrementalMode() await agilitySyncClient.runSync(); } - console.log(`Agility CMS => Getting page props for '${path}'...`); + console.log(`AgilityCMS => Getting page props for '${path}'...`); //get sitemap const sitemap = await agilitySyncClient.store.getSitemap({ channelName, languageCode }); @@ -135,7 +138,7 @@ const getAgilityPageProps = async ({ params, preview, locale }:any):Promise Fetching additional data via getCustomInitialProps for ${moduleItem.module}...`); + console.log(`AgilityCMS => Fetching additional data via getCustomInitialProps for ${moduleItem.module}...`); } const moduleData = await moduleInitializer({ @@ -183,28 +186,28 @@ const getAgilityPageProps = async ({ params, preview, locale }:any):Promise { +const getAgilityPaths = async (preview:boolean|undefined) => { - console.log(`Agility CMS => Fetching sitemap for getAgilityPaths...`); + console.log(`AgilityCMS => Fetching sitemap for getAgilityPaths...`); //determine if we are in preview mode - const isPreview = isDevelopmentMode; + const isPreview = isDevelopmentMode || preview; + + //determine if we've already done a full build yet + const buildFilePath = `${process.cwd()}/.next/cache/agility/build.log` + const isBuildComplete = fs.existsSync(buildFilePath) const agilitySyncClient = getSyncClient({ isPreview, - isDevelopmentMode + isDevelopmentMode, + isIncremental: isBuildComplete }); - //always sync to get latest if (! agilitySyncClient) { - console.log("Agility CMS => Sync client could not be accessed.") + console.log("AgilityCMS => Sync client could not be accessed.") return []; } - if (!isDevelopmentMode) { - console.log(`Agility CMS => Syncing ${isPreview ? "Preview" : "Live"} Mode`) - await agilitySyncClient.runSync(); - } const sitemapFlat = await agilitySyncClient.store.getSitemap({ channelName, @@ -214,7 +217,7 @@ const getAgilityPaths = async () => { if (!sitemapFlat) { - console.warn("Agility CMS => No Site map found. Make sure your environment variables are setup correctly.") + console.warn("AgilityCMS => No Site map found. Make sure your environment variables are setup correctly.") return [] } diff --git a/framework/agility/agility.sync.js b/framework/agility/agility.sync.js index cf8641161..6f861065c 100644 --- a/framework/agility/agility.sync.js +++ b/framework/agility/agility.sync.js @@ -1,8 +1,5 @@ -/* -THIS FILE IS ONLY EXECUTED LOCALLY -WHEN DOING A LOCAL SYNC ON DEMAND -IN DEVELOPMENT MODE -*/ + +const fs = require('fs') require("dotenv").config({ path: `.env.local`, @@ -12,22 +9,44 @@ const { getSyncClient } = require('./agility.config') const runSync = async () => { + setBuildLog(false) const agilitySyncClient = getSyncClient({ isPreview: true, isDevelopmentMode: true }) if (! agilitySyncClient) { - console.log("Agility CMS => Sync client could not be accessed.") + console.log("AgilityCMS => Sync client could not be accessed.") return; } await agilitySyncClient.runSync(); } -const syncAll = async () => { +const setBuildLog = (builtYN) => { + //clear out a file saying WE HAVE SYNC'D + const rootPath = process.cwd() + const filePath = `${rootPath}/.next/cache/agility/build.log` + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + + if (builtYN) { + //write out the build log so we know that we are up to date + fs.writeFileSync(filePath, "BUILT"); + } else { + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + } +} + +const preBuild = async () => { + + //clear the build log + setBuildLog(false) //sync preview mode let agilitySyncClient = getSyncClient({ isPreview: true, isDevelopmentMode: false }) if (! agilitySyncClient) { - console.log("Agility CMS => Sync client could not be accessed.") + console.log("AgilityCMS => Sync client could not be accessed.") return; } @@ -36,18 +55,27 @@ const syncAll = async () => { //sync production mode agilitySyncClient = getSyncClient({ isPreview: false, isDevelopmentMode: false }) if (! agilitySyncClient) { - console.log("Agility CMS => Sync client could not be accessed.") + console.log("AgilityCMS => Sync client could not be accessed.") return; } await agilitySyncClient.runSync(); + + +} + +const postBuild = async() => { + //mark the build log as BUILT + setBuildLog(true) } const clearSync = async () => { + setBuildLog(false) + const agilitySyncClient = getSyncClient({ isPreview: true, isDevelopmentMode: true }) if (! agilitySyncClient) { - console.log("Agility CMS => Sync client could not be accessed.") + console.log("AgilityCMS => Sync client could not be accessed.") return; } await agilitySyncClient.clearSync(); @@ -63,10 +91,13 @@ if (process.argv[2]) { //run the sync return runSync() - } else if (process.argv[2] === "sync-all") { - //sync both staging and live content - return syncAll() + } else if (process.argv[2] === "prebuild") { + //pre build actions + return preBuild() + } else if (process.argv[2] === "postbuild") { + //post build actions + return postBuild() } } diff --git a/framework/agility/next.file.sync.js b/framework/agility/next.file.sync.js new file mode 100644 index 000000000..74241c957 --- /dev/null +++ b/framework/agility/next.file.sync.js @@ -0,0 +1,153 @@ +const fs = require('fs') +const os = require('os') +const path = require('path') + +const { lockSync, unlockSync, checkSync, check } = require("proper-lockfile") + +require("dotenv").config({ + path: `.env.${process.env.NODE_ENV}`, +}) + +const sleep = (ms) => { + return new Promise(resolve => setTimeout(resolve, ms)); +} + + +const getFilePath = ({ options, itemType, languageCode, itemID }) => { + + const fileName = `${itemID}.json`; + return path.join(options.rootPath, languageCode, itemType, fileName); +} + +const saveItem = async ({ options, item, itemType, languageCode, itemID }) => { + + let filePath = getFilePath({ options, itemType, languageCode, itemID }); + + let dirPath = path.dirname(filePath); + + + if (!fs.existsSync(dirPath)) { + fs.mkdirSync(dirPath, { recursive: true }); + } + + let json = JSON.stringify(item); + fs.writeFileSync(filePath, json); +} + +const deleteItem = async ({ options, itemType, languageCode, itemID }) => { + + let filePath = getFilePath({ options, itemType, languageCode, itemID }); + + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + +} + +const mergeItemToList = async ({ options, item, languageCode, itemID, referenceName, definitionName }) => { + + let contentList = await getItem({ options, itemType: "list", languageCode, itemID: referenceName }); + + if (contentList == null) { + //initialize the list + contentList = [item]; + } else { + //replace the item... + const cIndex = contentList.findIndex((ci) => { + return ci.contentID === itemID; + }); + + if (item.properties.state === 3) { + //*** deleted item (remove from the list) *** + if (cIndex >= 0) { + //remove the item + contentList.splice(cIndex, 1); + } + + } else { + //*** regular item (merge) *** + if (cIndex >= 0) { + //replace the existing item + contentList[cIndex] = item; + } else { + //and it to the end of the + contentList.push(item); + } + } + } + + await saveItem({ options, item: contentList, itemType: "list", languageCode, itemID: referenceName }); +} + +const getItem = async ({ options, itemType, languageCode, itemID }) => { + let filePath = getFilePath({ options, itemType, languageCode, itemID }); + + if (!fs.existsSync(filePath)) return null; + + let json = fs.readFileSync(filePath, 'utf8'); + return JSON.parse(json); +} + +const clearItems = async ({ options }) => { + fs.rmdirSync(options.rootPath, { recursive: true }) +} + +const waitOnLock = async (lockFile) => { + + while (await check(lockFile)) { + await sleep(100) + } + +} + +const mutexLock = async () => { + + + const dir = os.tmpdir(); + const lockFile = `${dir}/${"agility-sync"}.mutex` + if (! fs.existsSync(lockFile)) { + fs.writeFileSync(lockFile, "agility-sync"); + } + + //THE LOCK IS ALREADY HELD - WAIT UP! + await waitOnLock(lockFile) + + try { + return lockSync(lockFile) + } catch (err) { + if (`${err}`.indexOf("Lock file is already being held") !== -1) { + + //this error happens when 2 processes try to get a lock at the EXACT same time (very rare) + await sleep(100) + await waitOnLock(lockFile) + + try { + return lockSync(lockFile) + } catch (e2) { + if (`${err}`.indexOf("Lock file is already being held") !== -1) { + + //this error happens when 2 processes try to get a lock at the EXACT same time (very rare) + await sleep(100) + await waitOnLock(lockFile) + return lockSync(lockFile) + } + } + } + + throw Error("The mutex lock could not be obtained.") + } + +} + + + + + +module.exports = { + saveItem, + deleteItem, + mergeItemToList, + getItem, + clearItems, + mutexLock +} \ No newline at end of file diff --git a/package.json b/package.json index 2ba95549b..d41c244b1 100644 --- a/package.json +++ b/package.json @@ -2,9 +2,11 @@ "name": "nextjs-commerce", "version": "1.0.0", "scripts": { - "dev": "next dev", - "prebuild": "node framework/agility/agility.sync.js sync-all", + "predev": "node framework/agility/agility.sync.js sync", + "dev": "next dev", + "prebuild": "node framework/agility/agility.sync.js prebuild", "build": "next build", + "postbuild": "node framework/agility/agility.sync.js postbuild", "start": "next start", "analyze": "BUNDLE_ANALYZE=both yarn build", "find:unused": "next-unused", @@ -56,6 +58,7 @@ "classnames": "^2.2.6", "cookie": "^0.4.1", "email-validator": "^2.0.4", + "fs-extra": "^9.0.1", "js-cookie": "^2.2.1", "keen-slider": "^5.2.4", "lodash.debounce": "^4.0.8", diff --git a/pages/[...slug].tsx b/pages/[...slug].tsx index 8ba31a72f..2f4980c0c 100644 --- a/pages/[...slug].tsx +++ b/pages/[...slug].tsx @@ -52,7 +52,7 @@ export async function getStaticProps({ preview, params, locale }: GetStaticProps } } - const pages = await getAgilityPaths() + const pages = await getAgilityPaths(preview) if (!page) { // We throw to make sure this fails at build time as this is never expected to happen @@ -68,7 +68,7 @@ export async function getStaticProps({ preview, params, locale }: GetStaticProps export async function getStaticPaths({ locales }: GetStaticPathsContext) { //get the paths configured in agility - let agilityPaths = await getAgilityPaths() + let agilityPaths = await getAgilityPaths(false) //remove product/product-details from the agility paths (special details page...) agilityPaths = agilityPaths.filter(p => p !== "/product/product-details") diff --git a/yarn.lock b/yarn.lock index e4bcb5dc1..edd8342ee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1750,6 +1750,11 @@ asynckit@^0.4.0: resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= +at-least-node@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2" + integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg== + atob@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" @@ -3541,6 +3546,16 @@ fs-extra@^8.0.0: jsonfile "^4.0.0" universalify "^0.1.0" +fs-extra@^9.0.1: + version "9.0.1" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.0.1.tgz#910da0062437ba4c39fedd863f1675ccfefcb9fc" + integrity sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ== + dependencies: + at-least-node "^1.0.0" + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^1.0.0" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -4391,6 +4406,15 @@ jsonfile@^4.0.0: optionalDependencies: graceful-fs "^4.1.6" +jsonfile@^6.0.1: + version "6.1.0" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" + integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== + dependencies: + universalify "^2.0.0" + optionalDependencies: + graceful-fs "^4.1.6" + jsonify@~0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" @@ -7423,6 +7447,16 @@ universalify@^0.1.0: resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== +universalify@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-1.0.0.tgz#b61a1da173e8435b2fe3c67d29b9adf8594bd16d" + integrity sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug== + +universalify@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" + integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ== + unixify@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unixify/-/unixify-1.0.0.tgz#3a641c8c2ffbce4da683a5c70f03a462940c2090"