optimizations

src/background.ts

@@ -1087,6 +1087,89 @@ export async function getPublicKey(receiver) {
   }
 }
 
+const MAX_STORAGE_SIZE = 3 * 1024 * 1024; // 3MB in bytes
+
+
+async function getDataPublishes(groupId, type) {
+  const wallet = await getSaveWallet();
+  const address = wallet.address0;
+
+  return new Promise((resolve) => {
+    chrome.storage.local.get([`${address}-publishData`], (result) => {
+      if (chrome.runtime.lastError) {
+        console.error('Error retrieving data:', chrome.runtime.lastError);
+        resolve(null); // Return null in case of an error
+        return;
+      }
+
+      let storedData = result[`${address}-publishData`] || {}; // Get the stored data or initialize an empty object
+      let groupData = storedData[groupId] || {}; // Get data by groupId
+      let typeData = groupData[type] || {}; // Get data by type
+
+      resolve(typeData); // Resolve with the data inside the specific type
+    });
+  });
+}
+
+
+async function addDataPublishes(newData, groupId, type) {
+  const wallet = await getSaveWallet();
+  const address = wallet.address0;
+  const nameIdentifier = `${newData.name}-${newData.identifier}`;
+
+  // Prevent adding data larger than 50KB
+  if (newData?.size > 50000) return false;
+
+  return new Promise((res) => {
+    chrome.storage.local.get([`${address}-publishData`], (result) => {
+      let storedData = result[`${address}-publishData`] || {}; // Get existing data or initialize
+      let groupData = storedData[groupId] || {}; // Get or initialize group by groupId
+      let typeData = groupData[type] || {}; // Get or initialize the type within the group
+
+      let totalSize = 0;
+
+      // Calculate the current size of all stored data
+      Object.values(storedData).forEach(group => {
+        Object.values(group).forEach(type => {
+          Object.values(type).forEach(data => {
+            totalSize += data.size; // Accumulate the sizes of actual data
+          });
+        });
+      });
+
+      // Check if adding the new data exceeds 3MB
+      if (totalSize + newData.size > MAX_STORAGE_SIZE) {
+        // Sort and remove older data within the group and type
+        let dataEntries = Object.entries(typeData);
+        dataEntries.sort((a, b) => a[1].timestampSaved - b[1].timestampSaved);
+
+        // Remove old data until there's enough space
+        while (totalSize + newData.size > MAX_STORAGE_SIZE && dataEntries.length > 0) {
+          const removedEntry = dataEntries.shift();
+          totalSize -= removedEntry[1].size;
+          delete typeData[removedEntry[0]]; // Remove from the typeData
+        }
+      }
+
+      // Add or update the new data within the group and type
+      if (totalSize + newData.size <= MAX_STORAGE_SIZE) {
+        typeData[`${nameIdentifier}`] = newData; // Add new data under name-identifier
+        groupData[type] = typeData; // Update type data within the group
+        storedData[groupId] = groupData; // Update group data within the stored data
+
+        // Save the updated structure back to chrome.storage.local
+        chrome.storage.local.set({ [`${address}-publishData`]: storedData }, () => {
+          res(true); // Data successfully added
+        });
+      } else {
+        console.error('Failed to add data, still exceeds storage limit.');
+        res(false); // Failure due to storage limit
+      }
+    });
+  });
+}
+
+
 async function decryptWallet({ password, wallet, walletVersion }) {
   try {
     const response = await decryptStoredWallet(password, wallet);
@@ -2713,7 +2796,34 @@ chrome?.runtime?.onMessage.addListener((request, sender, sendResponse) => {
                   });
               }
       
+
               break;
+              case "addDataPublishes":
+              {
+                const { data, groupId, type } = request.payload;
+                addDataPublishes(data, groupId, type)
+                  .then((res) => {
+                    sendResponse(res);
+                  })
+                  .catch((error) => {
+                    sendResponse({ error: error.message });
+                    console.error(error.message);
+                  });
+              }
+              break;
+              case "getDataPublishes":
+                {
+                  const { groupId, type } = request.payload;
+                  getDataPublishes(groupId, type)
+                    .then((res) => {
+                      sendResponse(res);
+                    })
+                    .catch((error) => {
+                      sendResponse({ error: error.message });
+                      console.error(error.message);
+                    });
+                }
+                break;
               case "cancelBan":
               {
                 const { groupId, qortalAddress } = request.payload;
@@ -3704,8 +3814,8 @@ chrome?.runtime?.onMessage.addListener((request, sender, sendResponse) => {
           const address = wallet.address0;
           const key1 = `tempPublish-${address}`
           const key2 = `group-data-${address}`
-
+          const key3 = `${address}-publishData`
-          chrome.storage.local.remove(["keyPair", "walletInfo", "apiKey", "active-groups-directs", key1, key2], () => {
+          chrome.storage.local.remove(["keyPair", "walletInfo", "apiKey", "active-groups-directs", key1, key2, key3], () => {
             if (chrome.runtime.lastError) {
               // Handle error
               console.error(chrome.runtime.lastError.message);

src/components/Chat/AnnouncementItem.tsx

@@ -21,7 +21,7 @@ export const AnnouncementItem = ({ message, messageData, setSelectedAnnouncement
 
         // dispatch(setIsLoadingGlobal(true))
         const identifier = `cm-${message.identifier}`;
-        const url = `${getBaseApiReact()}/arbitrary/resources/search?mode=ALL&service=DOCUMENT&identifier=${identifier}&limit=0&includemetadata=false&offset=${offset}&reverse=true`;
+        const url = `${getBaseApiReact()}/arbitrary/resources/search?mode=ALL&service=DOCUMENT&identifier=${identifier}&limit=0&includemetadata=false&offset=${offset}&reverse=true&prefix=true`;
        
         const response =  await requestQueueCommentCount.enqueue(() => { 
           return fetch(url, {

src/components/Chat/CreateCommonSecret.tsx

@@ -20,7 +20,7 @@ export const CreateCommonSecret = ({groupId, secretKey, isOwner,  myAddress, sec
     const queryString = admins.map((name) => `name=${name}`).join("&");
     const url = `${getBaseApiReact()}/arbitrary/resources/search?mode=ALL&service=DOCUMENT_PRIVATE&identifier=symmetric-qchat-group-${
       groupId
-    }&exactmatchnames=true&limit=0&reverse=true&${queryString}`;
+    }&exactmatchnames=true&limit=0&reverse=true&${queryString}&prefix=true`;
     const response = await fetch(url);
     if(!response.ok){
       throw new Error('network error')

src/components/Chat/GroupAnnouncements.tsx

@@ -31,6 +31,7 @@ import { AnnouncementDiscussion } from "./AnnouncementDiscussion";
 import { MyContext, getBaseApiReact, isMobile, pauseAllQueues, resumeAllQueues } from "../../App";
 import { RequestQueueWithPromise } from "../../utils/queue/queue";
 import { CustomizedSnackbars } from "../Snackbar/Snackbar";
+import {  addDataPublishesFunc, getDataPublishesFunc } from "../Group/Group";
 
 export const requestQueueCommentCount = new RequestQueueWithPromise(3)
 export const requestQueuePublishedAccouncements = new RequestQueueWithPromise(3)
@@ -145,20 +146,34 @@ export const GroupAnnouncements = ({
   const hasInitialized = useRef(false);
   const hasInitializedWebsocket = useRef(false);
   const editorRef = useRef(null);
-
+  const dataPublishes = useRef({})
   const setEditorRef = (editorInstance) => {
     editorRef.current = editorInstance;
   };
 
-  const getAnnouncementData = async ({ identifier, name }) => {
+  useEffect(()=> {
-    try {
+    if(!selectedGroup) return
+    (async ()=> {
+      const res = await getDataPublishesFunc(selectedGroup, 'anc')
+      dataPublishes.current = res || {}
+    })()
+  }, [selectedGroup])
+
+  const getAnnouncementData = async ({ identifier, name, resource }) => {
+    try {
+      let data = dataPublishes.current[`${name}-${identifier}`]
+      if(!data || (data?.update || data?.created !== (resource?.updated || resource?.created))){
+        const res = await requestQueuePublishedAccouncements.enqueue(()=> {
+          return fetch(
+            `${getBaseApiReact()}/arbitrary/DOCUMENT/${name}/${identifier}?encoding=base64`
+          );
+        }) 
+         data = await res.text();
+         await addDataPublishesFunc({...resource, data}, selectedGroup, 'anc')
+      } else {
+        data = data.data
+      }
     
-      const res = await requestQueuePublishedAccouncements.enqueue(()=> {
-        return fetch(
-          `${getBaseApiReact()}/arbitrary/DOCUMENT/${name}/${identifier}?encoding=base64`
-        );
-      }) 
-      const data = await res.text();
       const response = await decryptPublishes([{ data }], secretKey);
   
       const messageData = response[0];
@@ -169,7 +184,9 @@ export const GroupAnnouncements = ({
         };
       });
   
-    } catch (error) {}
+    } catch (error) {
+      console.log('error', error)
+    }
   };
 
  
@@ -282,7 +299,7 @@ export const GroupAnnouncements = ({
           secretKeyObject
         );
         const randomUid = uid.rnd();
-      const identifier = `grp-${selectedGroup}-anc-${randomUid}`;
+        const identifier = `grp-${selectedGroup}-anc-${randomUid}`;
         const res = await publishAnc({
           encryptedData: encryptSingle,
           identifier
@@ -335,7 +352,7 @@ export const GroupAnnouncements = ({
         setAnnouncements(responseData);
         setIsLoading(false);
         for (const data of responseData) {
-          getAnnouncementData({ name: data.name, identifier: data.identifier });
+          getAnnouncementData({ name: data.name, identifier: data.identifier, resource: data });
         }
       } catch (error) {
       } finally {

src/components/Group/Forum/GroupMail.tsx

@@ -55,6 +55,7 @@ import { executeEvent, subscribeToEvent, unsubscribeFromEvent } from "../../../u
 import RefreshIcon from '@mui/icons-material/Refresh';
 import { getBaseApiReact } from "../../../App";
 import { WrapperUserAction } from "../../WrapperUserAction";
+import { addDataPublishesFunc, getDataPublishesFunc } from "../Group";
 const filterOptions = ["Recently active", "Newest", "Oldest"];
 
 export const threadIdentifier = "DOCUMENT";
@@ -76,6 +77,7 @@ export const GroupMail = ({
   const [isOpenFilterList, setIsOpenFilterList] = useState<boolean>(false);
   const anchorElInstanceFilter = useRef<any>(null);
   const [tempPublishedList, setTempPublishedList] = useState([])
+  const dataPublishes = useRef({})
 
   const [isLoading, setIsLoading] = useState(false)
   const groupIdRef = useRef<any>(null);
@@ -83,6 +85,14 @@ export const GroupMail = ({
     return selectedGroup?.groupId;
   }, [selectedGroup]);
 
+  useEffect(()=> {
+    if(!groupId) return
+    (async ()=> {
+      const res = await getDataPublishesFunc(groupId, 'thread')
+      dataPublishes.current = res || {}
+    })()
+  }, [groupId])
+
   useEffect(() => {
     if (groupId !== groupIdRef?.current) {
       setCurrentThread(null);
@@ -110,12 +120,18 @@ export const GroupMail = ({
    
   }
 
-  const getEncryptedResource = async ({ name, identifier }) => {
+  const getEncryptedResource = async ({ name, identifier, resource }) => {
-  
+    let data = dataPublishes.current[`${name}-${identifier}`]
+    if(!data || (data?.update || data?.created !== (resource?.updated || resource?.created))){
     const res = await fetch(
       `${getBaseApiReact()}/arbitrary/DOCUMENT/${name}/${identifier}?encoding=base64`
     );
-    const data = await res.text();
+     data = await res.text();
+     await addDataPublishesFunc({...resource, data}, groupId, 'thread')
+
+    } else {
+      data = data.data
+    }
     const response = await decryptPublishes([{ data }], secretKey);
 
     const messageData = response[0];
@@ -190,6 +206,7 @@ export const GroupMail = ({
                 getEncryptedResource({
                   name: message.name,
                   identifier: message.identifier,
+                  resource: message
                 }),
                 delay(5000),
               ]);
@@ -309,6 +326,7 @@ export const GroupMail = ({
                   getEncryptedResource({
                     name: thread.name,
                     identifier: message.threadId,
+                    resource: thread
                   }),
                   delay(10000),
                 ]);

src/components/Group/Forum/Thread.tsx

@@ -22,7 +22,10 @@ import { subscribeToEvent, unsubscribeFromEvent } from "../../../utils/events";
 import RefreshIcon from "@mui/icons-material/Refresh";
 import { getBaseApiReact, isMobile } from "../../../App";
 import { ArrowDownward as ArrowDownwardIcon, ArrowUpward as ArrowUpwardIcon } from '@mui/icons-material';
-
+import { addDataPublishesFunc, getDataPublishesFunc } from "../Group";
+import { RequestQueueWithPromise } from "../../../utils/queue/queue";
+const requestQueueSaveToLocal = new RequestQueueWithPromise(1)
+const requestQueueDownloadPost = new RequestQueueWithPromise(3)
 interface ThreadProps {
   currentThread: any;
   groupInfo: any;
@@ -30,11 +33,22 @@ interface ThreadProps {
   members: any;
 }
 
-const getEncryptedResource = async ({ name, identifier, secretKey }) => {
+const getEncryptedResource = async ({ name, identifier, secretKey, resource, groupId, dataPublishes }) => {
-  const res = await fetch(
+  let data = dataPublishes[`${name}-${identifier}`]
-    `${getBaseApiReact()}/arbitrary/DOCUMENT/${name}/${identifier}?encoding=base64`
+    if(!data || (data?.update || data?.created !== (resource?.updated || resource?.created))){
-  );
+  const res = await requestQueueDownloadPost.enqueue(()=> {
-  const data = await res.text();
+    return fetch(
+      `${getBaseApiReact()}/arbitrary/DOCUMENT/${name}/${identifier}?encoding=base64`
+    );
+  }) 
+   data = await res.text();
+   await requestQueueSaveToLocal.enqueue(()=> {
+    return addDataPublishesFunc({...resource, data}, groupId, 'thmsg')
+   })
+    
+    } else {
+      data = data.data
+    }
   const response = await decryptPublishes([{ data }], secretKey);
 
   const messageData = response[0];
@@ -71,6 +85,18 @@ export const Thread = ({
   const secretKeyRef = useRef(null);
   const currentThreadRef = useRef(null);
   const containerRef = useRef(null);
+  const dataPublishes = useRef({})
+
+ 
+  const getSavedData = useCallback(async (groupId)=> {
+    const res = await getDataPublishesFunc(groupId, 'thmsg')
+    dataPublishes.current = res || {}
+  }, [])
+
+  useEffect(()=> {
+    if(!groupInfo?.groupId) return
+    getSavedData(groupInfo?.groupId)
+  }, [groupInfo?.groupId])
 
   useEffect(() => {
     currentThreadRef.current = currentThread;
@@ -86,6 +112,9 @@ export const Thread = ({
         identifier: message.identifier,
         name: message.name,
         secretKey,
+        resource: message,
+        groupId: groupInfo?.groupId,
+        dataPublishes: dataPublishes.current
       });
 
       const fullObject = {
@@ -129,7 +158,7 @@ export const Thread = ({
   }
 
   const getMailMessages = React.useCallback(
-    async (groupInfo: any, before, after, isReverse) => {
+    async (groupInfo: any, before, after, isReverse, groupId) => {
       try {
         setTempPublishedList([])
         setIsLoading(true);
@@ -169,7 +198,7 @@ export const Thread = ({
         }
         // let newMessages: any[] = []
         for (const message of responseData) {
-          getIndividualMsg(message);
+            getIndividualMsg(message);
         }
         setMessages(fullArrayMsg);
         if (before === null && after === null && isReverse) {
@@ -220,17 +249,18 @@ export const Thread = ({
           updateThreadActivityCurrentThread()
         }
       } catch (error) {
+        console.log('error', error)
       } finally {
         setIsLoading(false);
+        getSavedData(groupId)
       }
     },
     [messages, secretKey]
   );
   const getMessages = React.useCallback(async () => {
-
+    if (!currentThread || !secretKey || !groupInfo?.groupId) return;
-    if (!currentThread || !secretKey) return;
+    await getMailMessages(currentThread, null, null, false, groupInfo?.groupId);
-    await getMailMessages(currentThread, null, null, false);
+  }, [getMailMessages, currentThread, secretKey, groupInfo?.groupId]);
-  }, [getMailMessages, currentThread, secretKey]);
   const firstMount = useRef(false);
 
   const saveTimestamp = useCallback((currentThread: any, username?: string) => {
@@ -327,6 +357,9 @@ export const Thread = ({
               identifier: message.identifier,
               name: message.name,
               secretKey: secretKeyRef.current,
+              resource: message,
+              groupId: groupInfo?.groupId,
+              dataPublishes: dataPublishes.current
             });
 
             const fullObject = {
@@ -369,7 +402,7 @@ export const Thread = ({
   const threadFetchModeFunc = (e) => {
     const mode = e.detail?.mode;
     if (mode === "last-page") {
-      getMailMessages(currentThread, null, null, true);
+      getMailMessages(currentThread, null, null, true, groupInfo?.groupId);
     }
     firstMount.current = true;
   };
@@ -443,7 +476,6 @@ export const Thread = ({
     }
   };
 
-  console.log('showScrollButton', showScrollButton)
 
   if (!currentThread) return null;
   return (
@@ -566,7 +598,7 @@ export const Thread = ({
               textTransformation: 'capitalize'
             }}
               onClick={() => {
-                getMailMessages(currentThread, null, null, false);
+                getMailMessages(currentThread, null, null, false, groupInfo?.groupId);
               }}
               disabled={!hasFirstPage}
               variant="contained"
@@ -584,7 +616,8 @@ export const Thread = ({
                   currentThread,
                   messages[0].created,
                   null,
-                  false
+                  false,
+                  groupInfo?.groupId
                 );
               }}
               disabled={!hasPreviousPage}
@@ -603,7 +636,8 @@ export const Thread = ({
                   currentThread,
                   null,
                   messages[messages.length - 1].created,
-                  false
+                  false,
+                  groupInfo?.groupId
                 );
               }}
               disabled={!hasNextPage}
@@ -618,7 +652,7 @@ export const Thread = ({
               textTransformation: 'capitalize'
             }}
               onClick={() => {
-                getMailMessages(currentThread, null, null, true);
+                getMailMessages(currentThread, null, null, true, groupInfo?.groupId);
               }}
               disabled={!hasLastPage}
               variant="contained"
@@ -680,7 +714,7 @@ export const Thread = ({
                   variant="outlined"
                   startIcon={<RefreshIcon />}
                   onClick={() => {
-                    getMailMessages(currentThread, null, null, true);
+                    getMailMessages(currentThread, null, null, true, groupInfo?.groupId);
                   }}
                   sx={{
                     color: "white",
@@ -711,7 +745,7 @@ export const Thread = ({
               textTransformation: 'capitalize'
             }}
               onClick={() => {
-                getMailMessages(currentThread, null, null, false);
+                getMailMessages(currentThread, null, null, false, groupInfo?.groupId);
               }}
               disabled={!hasFirstPage}
               variant="contained"
@@ -729,7 +763,8 @@ export const Thread = ({
                   currentThread,
                   messages[0].created,
                   null,
-                  false
+                  false,
+                  groupInfo?.groupId
                 );
               }}
               disabled={!hasPreviousPage}
@@ -748,7 +783,8 @@ export const Thread = ({
                   currentThread,
                   null,
                   messages[messages.length - 1].created,
-                  false
+                  false,
+                  groupInfo?.groupId
                 );
               }}
               disabled={!hasNextPage}
@@ -763,7 +799,7 @@ export const Thread = ({
               textTransformation: 'capitalize'
             }}
               onClick={() => {
-                getMailMessages(currentThread, null, null, true);
+                getMailMessages(currentThread, null, null, true, groupInfo?.groupId);
               }}
               disabled={!hasLastPage}
               variant="contained"

src/components/Group/Group.tsx

@@ -203,6 +203,51 @@ export const decryptResource = async (data: string) => {
   } catch (error) {}
 };
 
+export const addDataPublishesFunc = async (data: string, groupId, type) => {
+  try {
+    return new Promise((res, rej) => {
+      chrome?.runtime?.sendMessage(
+        {
+          action: "addDataPublishes",
+          payload: {
+            data,
+            groupId,
+            type
+          },
+        },
+        (response) => {
+          if (!response?.error) {
+            res(response);
+          }
+          rej(response.error);
+        }
+      );
+    });
+  } catch (error) {}
+};
+
+export const getDataPublishesFunc = async (groupId, type) => {
+  try {
+    return new Promise((res, rej) => {
+      chrome?.runtime?.sendMessage(
+        {
+          action: "getDataPublishes",
+          payload: {
+            groupId,
+            type
+          },
+        },
+        (response) => {
+          if (!response?.error) {
+            res(response);
+          }
+          rej(response.error);
+        }
+      );
+    });
+  } catch (error) {}
+};
+
 export async function getNameInfo(address: string) {
   const response = await fetch(`${getBaseApiReact()}/names/address/` + address);
   const nameData = await response.json();
@@ -567,7 +612,7 @@ export const Group = ({
     const queryString = admins.map((name) => `name=${name}`).join("&");
     const url = `${getBaseApiReact()}/arbitrary/resources/search?mode=ALL&service=DOCUMENT_PRIVATE&identifier=symmetric-qchat-group-${
       selectedGroup?.groupId
-    }&exactmatchnames=true&limit=0&reverse=true&${queryString}`;
+    }&exactmatchnames=true&limit=0&reverse=true&${queryString}&prefix=true`;
     const response = await fetch(url);
     if (!response.ok) {
       throw new Error("network error");