From fc82f0b62222ba4ae80f0e8a8930f4de1b9914a9 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 11 Feb 2022 13:58:45 +0000
Subject: [PATCH] Use 5 builder threads, so that one slow resource (e.g. a
 thumbnail) doesn't hold up the other queued build items.

This can be replaced with a task-based approach longer term.
---
 .../ArbitraryDataBuildQueueItem.java          |  9 ++++-
 .../arbitrary/ArbitraryDataBuildManager.java  |  7 +++-
 .../arbitrary/ArbitraryDataBuilderThread.java | 40 ++++++++++---------
 3 files changed, 35 insertions(+), 21 deletions(-)

diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java
index ffbf8fe3..ddbf9f24 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java
@@ -27,13 +27,20 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
         this.creationTimestamp = NTP.getTime();
     }
 
+    public void prepareForBuild() {
+        this.buildStartTimestamp = NTP.getTime();
+    }
+
     public void build() throws IOException, DataException, MissingDataException {
         Long now = NTP.getTime();
         if (now == null) {
+            this.buildStartTimestamp = null;
             throw new DataException("NTP time hasn't synced yet");
         }
 
-        this.buildStartTimestamp = now;
+        if (this.buildStartTimestamp == null) {
+            this.buildStartTimestamp = now;
+        }
         ArbitraryDataReader arbitraryDataReader =
                 new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier);
 
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java
index d607047e..0054356e 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java
@@ -42,8 +42,11 @@ public class ArbitraryDataBuildManager extends Thread {
         try {
             // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
             // This can be expanded to have multiple threads processing the build queue when needed
-            ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1);
-            arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
+            int threadCount = 5;
+            ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(threadCount);
+            for (int i = 0; i < threadCount; i++) {
+                arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
+            }
 
             while (!isStopping) {
                 // Nothing to do yet
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java
index 1c03daed..17808daa 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java
@@ -36,37 +36,41 @@ public class ArbitraryDataBuilderThread implements Runnable {
                     continue;
                 }
 
-                Map.Entry<String, ArbitraryDataBuildQueueItem> next = null;
+                Long now = NTP.getTime();
+                if (now == null) {
+                    continue;
+                }
+
+                ArbitraryDataBuildQueueItem queueItem = null;
 
                 // Find resources that are queued for building
                 synchronized (buildManager.arbitraryDataBuildQueue) {
-                    next = buildManager.arbitraryDataBuildQueue
+                    Map.Entry<String, ArbitraryDataBuildQueueItem> next = buildManager.arbitraryDataBuildQueue
                             .entrySet().stream()
                             .filter(e -> e.getValue().isQueued())
                             .findFirst().orElse(null);
-                }
 
-                if (next == null) {
-                    continue;
-                }
+                    if (next == null) {
+                        continue;
+                    }
 
-                Long now = NTP.getTime();
-                if (now == null) {
-                    continue;
-                }
+                    queueItem = next.getValue();
 
-                ArbitraryDataBuildQueueItem queueItem = next.getValue();
+                    if (queueItem == null) {
+                        this.removeFromQueue(queueItem);
+                        continue;
+                    }
 
-                if (queueItem == null) {
-                    this.removeFromQueue(queueItem);
-                }
+                    // Ignore builds that have failed recently
+                    if (buildManager.isInFailedBuildsList(queueItem)) {
+                        this.removeFromQueue(queueItem);
+                        continue;
+                    }
 
-                // Ignore builds that have failed recently
-                if (buildManager.isInFailedBuildsList(queueItem)) {
-                    continue;
+                    // Set the start timestamp, to prevent other threads from building it at the same time
+                    queueItem.prepareForBuild();
                 }
 
-
                 try {
                     // Perform the build
                     LOGGER.info("Building {}...", queueItem);