tmp file management

framework/agility/agility.config.js

@@ -1,5 +1,8 @@
 const agilityContentSync = require("@agility/content-sync");
-const agilityFileSystem = require("@agility/content-sync/src/store-interface-filesystem");
+const nextFileSystem = require("./next.file.sync");
+
+const path = require("path")
+const fs = require('fs-extra');
 
 const agilityConfig = {
 	guid: process.env.AGILITY_GUID, //Set your guid here
@@ -10,26 +13,24 @@ const agilityConfig = {
 	securityKey: process.env.AGILITY_SECURITY_KEY, //the website security key used to validate and generate preview keys
 };
 
-const getSyncClient = ({ isPreview, isDevelopmentMode }) => {
+const getSyncClient = ({ isPreview, isDevelopmentMode, isIncremental }) => {
 
 	const rootPath = process.cwd()
 
-  let cachePath = `${rootPath}/.next/cache/agility/${
+	let cachePath = `${rootPath}/.next/cache/agility/${isPreview ? "preview" : "live" }`;
-    isPreview ? "preview" : "live"
-  }`;
 
-//   if (!isDevelopmentMode) {
+	//if we are in "incremental" mode, we need to use the tmp folder...
-//     //we are in prod and need to use a different directory that Vercel understands
+	if (isIncremental) {
-//     cachePath = `/tmp/agilitycache/${isPreview ? "preview" : "live"}`;
+		cachePath = `/tmp/agilitycache/${isPreview ? "preview" : "live"}`;
-//   }
+	}
 
-  console.log(`Agility CMS => Content cache path is ${cachePath}`);
+	console.log(`AgilityCMS => Content cache path is ${cachePath}`);
 	const apiKey = isPreview
 		? agilityConfig.previewAPIKey
 		: agilityConfig.fetchAPIKey;
 
 	if (!agilityConfig.guid) {
-    console.log("Agility CMS => No GUID was provided.");
+		console.log("AgilityCMS => No GUID was provided.");
 		return null;
 	}
 
@@ -40,15 +41,34 @@ const getSyncClient = ({ isPreview, isDevelopmentMode }) => {
 		languages: [agilityConfig.languageCode],
 		channels: [agilityConfig.channelName],
 		store: {
-      interface: agilityFileSystem,
+			interface: nextFileSystem,
 			options: {
-        rootPath: cachePath,
+				rootPath: cachePath
 			},
 		},
 	});
 };
 
+
+const prepIncrementalMode = async () => {
+
+	let cachePath = `${rootPath}/.next/cache/agility/`;
+	const tempPath = `/tmp/agilitycache/`;
+
+	const buildFilePath = path.join(tempPath, "build.log")
+
+	//check for the build file in here...
+	if (!fs.existsSync(buildFilePath)) {
+		console.log(`Copying Agility Content files to temp folder: ${tempPath}`)
+		//copy everything across from cachePath
+		await fs.copy(cachePath, tempPath)
+	}
+}
+
+
+
 module.exports = {
 	agilityConfig,
 	getSyncClient,
+	prepIncrementalMode
 };

framework/agility/agility.node.ts

@@ -1,10 +1,11 @@
+import fs from "fs"
 import crypto from 'crypto'
 import { asyncForEach } from "./utils"
 
 import { ModuleWithInit } from "@agility/types"
 
 //Agility API stuff
-import { agilityConfig, getSyncClient } from './agility.config'
+import { agilityConfig, getSyncClient, prepIncrementalMode } from './agility.config'
 import GlobalFooter from 'components/agility-global/GlobalFooter'
 import GlobalHeader from 'components/agility-global/GlobalHeader'
 
@@ -41,35 +42,37 @@ const getAgilityPageProps = async ({ params, preview, locale }:any):Promise<Agil
 		})
 	}
 
+	//determine if we've already done a full build yet
+	const buildFilePath = `${process.cwd()}/.next/cache/agility/build.log`
+	const isBuildComplete = fs.existsSync(buildFilePath)
+
 	//TODO: use locale to determin LANGUAGECODE (pulled from config at this point...)
 
 	//determine if we are in preview mode
 	const isPreview:boolean = (preview || isDevelopmentMode);
 
-
-
 	const agilitySyncClient = getSyncClient({
 		isPreview: isPreview,
-		isDevelopmentMode
+		isDevelopmentMode,
+		isIncremental: isBuildComplete
 	});
 
-
-	//always sync to get latest
-
-
 	if (! agilitySyncClient) {
-		console.log("Agility CMS => Sync client could not be accessed.")
+		console.log("AgilityCMS => Sync client could not be accessed.")
 		return {
 			notFound: true
 		};
 	}
 
-	if (!isDevelopmentMode) {
+	if (preview || isBuildComplete) {
-		console.log(`Agility CMS => Syncing ${isPreview ? "Preview" : "Live"} Mode`)
+		//only do on-demand sync in next's preview mode or incremental build...
+		console.log(`AgilityCMS => Sync On-demand ${isPreview ? "Preview" : "Live"} Mode`)
+
+		await prepIncrementalMode()
 		await agilitySyncClient.runSync();
 	}
 
-	console.log(`Agility CMS => Getting page props for '${path}'...`);
+	console.log(`AgilityCMS => Getting page props for '${path}'...`);
 
 	//get sitemap
 	const sitemap = await agilitySyncClient.store.getSitemap({ channelName, languageCode });
@@ -135,7 +138,7 @@ const getAgilityPageProps = async ({ params, preview, locale }:any):Promise<Agil
 
 				//we have some additional data in the module we'll need, execute that method now, so it can be included in SSG
 				if (isDevelopmentMode) {
-					console.log(`Agility CMS => Fetching additional data via getCustomInitialProps for ${moduleItem.module}...`);
+					console.log(`AgilityCMS => Fetching additional data via getCustomInitialProps for ${moduleItem.module}...`);
 				}
 
 				const moduleData = await moduleInitializer({
@@ -183,28 +186,28 @@ const getAgilityPageProps = async ({ params, preview, locale }:any):Promise<Agil
 	}
 }
 
-const getAgilityPaths = async () => {
+const getAgilityPaths = async (preview:boolean|undefined) => {
 
-	console.log(`Agility CMS => Fetching sitemap for getAgilityPaths...`);
+	console.log(`AgilityCMS => Fetching sitemap for getAgilityPaths...`);
 
 	//determine if we are in preview mode
-	const isPreview = isDevelopmentMode;
+	const isPreview = isDevelopmentMode || preview;
+
+	//determine if we've already done a full build yet
+	const buildFilePath = `${process.cwd()}/.next/cache/agility/build.log`
+	const isBuildComplete = fs.existsSync(buildFilePath)
 
 	const agilitySyncClient = getSyncClient({
 		isPreview,
-		isDevelopmentMode
+		isDevelopmentMode,
+		isIncremental: isBuildComplete
 	});
 
-	//always sync to get latest
 
 	if (! agilitySyncClient) {
-		console.log("Agility CMS => Sync client could not be accessed.")
+		console.log("AgilityCMS => Sync client could not be accessed.")
 		return [];
 	}
-	if (!isDevelopmentMode) {
-		console.log(`Agility CMS => Syncing ${isPreview ? "Preview" : "Live"} Mode`)
-		await agilitySyncClient.runSync();
-	}
 
 	const sitemapFlat = await agilitySyncClient.store.getSitemap({
 		channelName,
@@ -214,7 +217,7 @@ const getAgilityPaths = async () => {
 
 
 	if (!sitemapFlat) {
-		console.warn("Agility CMS => No Site map found.  Make sure your environment variables are setup correctly.")
+		console.warn("AgilityCMS => No Site map found.  Make sure your environment variables are setup correctly.")
 		return []
 	}
 

framework/agility/agility.sync.js

@@ -1,8 +1,5 @@
-/*
+
-THIS FILE IS ONLY EXECUTED LOCALLY
+const fs = require('fs')
-WHEN DOING A LOCAL SYNC ON DEMAND
-IN DEVELOPMENT MODE
-*/
 
 require("dotenv").config({
 	path: `.env.local`,
@@ -12,22 +9,44 @@ const { getSyncClient } = require('./agility.config')
 
 
 const runSync = async () => {
+	setBuildLog(false)
 
 	const agilitySyncClient = getSyncClient({ isPreview: true, isDevelopmentMode: true })
 	if (! agilitySyncClient) {
-		console.log("Agility CMS => Sync client could not be accessed.")
+		console.log("AgilityCMS => Sync client could not be accessed.")
 		return;
 	}
 
 	await agilitySyncClient.runSync();
 }
 
-const syncAll = async () => {
+const setBuildLog = (builtYN) => {
+	//clear out a file saying WE HAVE SYNC'D
+	const rootPath = process.cwd()
+	const filePath = `${rootPath}/.next/cache/agility/build.log`
+	if (fs.existsSync(filePath)) {
+		fs.unlinkSync(filePath);
+	}
+
+	if (builtYN) {
+		//write out the build log so we know that we are up to date
+		fs.writeFileSync(filePath, "BUILT");
+	} else {
+		if (fs.existsSync(filePath)) {
+			fs.unlinkSync(filePath);
+		}
+	}
+}
+
+const preBuild = async () => {
+
+	//clear the build log
+	setBuildLog(false)
 
 	//sync preview mode
 	let agilitySyncClient = getSyncClient({ isPreview: true, isDevelopmentMode: false })
 	if (! agilitySyncClient) {
-		console.log("Agility CMS => Sync client could not be accessed.")
+		console.log("AgilityCMS => Sync client could not be accessed.")
 		return;
 	}
 
@@ -36,18 +55,27 @@ const syncAll = async () => {
 	//sync production mode
 	agilitySyncClient = getSyncClient({ isPreview: false, isDevelopmentMode: false })
 	if (! agilitySyncClient) {
-		console.log("Agility CMS => Sync client could not be accessed.")
+		console.log("AgilityCMS => Sync client could not be accessed.")
 		return;
 	}
 
 	await agilitySyncClient.runSync();
+
+
+}
+
+const postBuild = async() => {
+	//mark the build log as BUILT
+	setBuildLog(true)
 }
 
 const clearSync = async () => {
 
+	setBuildLog(false)
+
 	const agilitySyncClient = getSyncClient({ isPreview: true, isDevelopmentMode: true })
 	if (! agilitySyncClient) {
-		console.log("Agility CMS => Sync client could not be accessed.")
+		console.log("AgilityCMS => Sync client could not be accessed.")
 		return;
 	}
 	await agilitySyncClient.clearSync();
@@ -63,10 +91,13 @@ if (process.argv[2]) {
 		//run the sync
 		return runSync()
 
-	} else if (process.argv[2] === "sync-all") {
+	} else if (process.argv[2] === "prebuild") {
-		//sync both staging and live content
+		//pre build actions
-		return syncAll()
+		return preBuild()
 
+	} else if (process.argv[2] === "postbuild") {
+		//post build actions
+		return postBuild()
 	}
 }
 

framework/agility/next.file.sync.js

@@ -0,0 +1,153 @@
+const fs = require('fs')
+const os = require('os')
+const path = require('path')
+
+const { lockSync, unlockSync, checkSync, check }  = require("proper-lockfile")
+
+require("dotenv").config({
+	path: `.env.${process.env.NODE_ENV}`,
+})
+
+const sleep = (ms) => {
+	return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+
+const getFilePath = ({ options, itemType, languageCode, itemID }) => {
+
+	const fileName = `${itemID}.json`;
+	return path.join(options.rootPath, languageCode, itemType, fileName);
+}
+
+const saveItem = async ({ options, item, itemType, languageCode, itemID }) => {
+
+	let filePath = getFilePath({ options, itemType, languageCode, itemID });
+
+	let dirPath = path.dirname(filePath);
+
+
+	if (!fs.existsSync(dirPath)) {
+		fs.mkdirSync(dirPath, { recursive: true });
+	}
+
+	let json = JSON.stringify(item);
+	fs.writeFileSync(filePath, json);
+}
+
+const deleteItem = async ({ options, itemType, languageCode, itemID }) => {
+
+	let filePath = getFilePath({ options, itemType, languageCode, itemID });
+
+	if (fs.existsSync(filePath)) {
+		fs.unlinkSync(filePath);
+	}
+
+}
+
+const mergeItemToList = async ({ options, item, languageCode, itemID, referenceName, definitionName }) => {
+
+	let contentList = await getItem({ options, itemType: "list", languageCode, itemID: referenceName });
+
+	if (contentList == null) {
+		//initialize the list
+		contentList = [item];
+	} else {
+		//replace the item...
+		const cIndex = contentList.findIndex((ci) => {
+			return ci.contentID === itemID;
+		});
+
+		if (item.properties.state === 3) {
+			//*** deleted item (remove from the list) ***
+			if (cIndex >= 0) {
+				//remove the item
+				contentList.splice(cIndex, 1);
+			}
+
+		} else {
+			//*** regular item (merge) ***
+			if (cIndex >= 0) {
+				//replace the existing item
+				contentList[cIndex] = item;
+			} else {
+				//and it to the end of the
+				contentList.push(item);
+			}
+		}
+	}
+
+	await saveItem({ options, item: contentList, itemType: "list", languageCode, itemID: referenceName });
+}
+
+const getItem = async ({ options, itemType, languageCode, itemID }) => {
+	let filePath = getFilePath({ options, itemType, languageCode, itemID });
+
+	if (!fs.existsSync(filePath)) return null;
+
+	let json = fs.readFileSync(filePath, 'utf8');
+	return JSON.parse(json);
+}
+
+const clearItems = async ({ options }) => {
+	fs.rmdirSync(options.rootPath, { recursive: true })
+}
+
+const waitOnLock = async (lockFile) => {
+
+	while (await check(lockFile)) {
+		await sleep(100)
+	}
+
+}
+
+const mutexLock = async () => {
+
+
+	const dir = os.tmpdir();
+	const lockFile = `${dir}/${"agility-sync"}.mutex`
+	if (! fs.existsSync(lockFile)) {
+		fs.writeFileSync(lockFile, "agility-sync");
+	}
+
+	//THE LOCK IS ALREADY HELD - WAIT UP!
+	await waitOnLock(lockFile)
+
+	try {
+		return lockSync(lockFile)
+	} catch (err) {
+		if (`${err}`.indexOf("Lock file is already being held") !== -1) {
+
+			//this error happens when 2 processes try to get a lock at the EXACT same time (very rare)
+			await sleep(100)
+			await waitOnLock(lockFile)
+
+			try {
+				return lockSync(lockFile)
+			} catch (e2) {
+				if (`${err}`.indexOf("Lock file is already being held") !== -1) {
+
+					//this error happens when 2 processes try to get a lock at the EXACT same time (very rare)
+					await sleep(100)
+					await waitOnLock(lockFile)
+					return lockSync(lockFile)
+				}
+			}
+		}
+
+		throw Error("The mutex lock could not be obtained.")
+	}
+
+}
+
+
+
+
+
+module.exports = {
+	saveItem,
+	deleteItem,
+	mergeItemToList,
+	getItem,
+	clearItems,
+	mutexLock
+}

package.json

@@ -2,9 +2,11 @@
   "name": "nextjs-commerce",
   "version": "1.0.0",
   "scripts": {
+    "predev": "node framework/agility/agility.sync.js sync",
     "dev": "next dev",
-	"prebuild": "node framework/agility/agility.sync.js sync-all",
+    "prebuild": "node framework/agility/agility.sync.js prebuild",
     "build": "next build",
+    "postbuild": "node framework/agility/agility.sync.js postbuild",
     "start": "next start",
     "analyze": "BUNDLE_ANALYZE=both yarn build",
     "find:unused": "next-unused",
@@ -56,6 +58,7 @@
     "classnames": "^2.2.6",
     "cookie": "^0.4.1",
     "email-validator": "^2.0.4",
+    "fs-extra": "^9.0.1",
     "js-cookie": "^2.2.1",
     "keen-slider": "^5.2.4",
     "lodash.debounce": "^4.0.8",

pages/[...slug].tsx

@@ -52,7 +52,7 @@ export async function getStaticProps({ preview, params, locale }: GetStaticProps
 		}
 	}
 
-	const pages = await getAgilityPaths()
+	const pages = await getAgilityPaths(preview)
 
 	if (!page) {
 		// We throw to make sure this fails at build time as this is never expected to happen
@@ -68,7 +68,7 @@ export async function getStaticProps({ preview, params, locale }: GetStaticProps
 export async function getStaticPaths({ locales }: GetStaticPathsContext) {
 
 	//get the paths configured in agility
-	let agilityPaths = await getAgilityPaths()
+	let agilityPaths = await getAgilityPaths(false)
 
 	//remove product/product-details from the agility paths (special details page...)
 	agilityPaths = agilityPaths.filter(p => p !== "/product/product-details")

yarn.lock

@@ -1750,6 +1750,11 @@ asynckit@^0.4.0:
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
   integrity sha1-x57Zf380y48robyXkLzDZkdLS3k=
 
+at-least-node@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2"
+  integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==
+
 atob@^2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9"
@@ -3541,6 +3546,16 @@ fs-extra@^8.0.0:
     jsonfile "^4.0.0"
     universalify "^0.1.0"
 
+fs-extra@^9.0.1:
+  version "9.0.1"
+  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.0.1.tgz#910da0062437ba4c39fedd863f1675ccfefcb9fc"
+  integrity sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==
+  dependencies:
+    at-least-node "^1.0.0"
+    graceful-fs "^4.2.0"
+    jsonfile "^6.0.1"
+    universalify "^1.0.0"
+
 fs.realpath@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
@@ -4391,6 +4406,15 @@ jsonfile@^4.0.0:
   optionalDependencies:
     graceful-fs "^4.1.6"
 
+jsonfile@^6.0.1:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae"
+  integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==
+  dependencies:
+    universalify "^2.0.0"
+  optionalDependencies:
+    graceful-fs "^4.1.6"
+
 jsonify@~0.0.0:
   version "0.0.0"
   resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73"
@@ -7423,6 +7447,16 @@ universalify@^0.1.0:
   resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
   integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==
 
+universalify@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/universalify/-/universalify-1.0.0.tgz#b61a1da173e8435b2fe3c67d29b9adf8594bd16d"
+  integrity sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==
+
+universalify@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717"
+  integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==
+
 unixify@1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/unixify/-/unixify-1.0.0.tgz#3a641c8c2ffbce4da683a5c70f03a462940c2090"