Browse Source

Use 5 builder threads, so that one slow resource (e.g. a thumbnail) doesn't hold up the other queued build items.

This can be replaced with a task-based approach longer term.
online-level-zero-accounts-api-call
CalDescent 3 years ago
parent
commit
fc82f0b622
  1. 7
      src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java
  2. 5
      src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java
  3. 22
      src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java

7
src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java

@ -27,13 +27,20 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
this.creationTimestamp = NTP.getTime(); this.creationTimestamp = NTP.getTime();
} }
public void prepareForBuild() {
this.buildStartTimestamp = NTP.getTime();
}
public void build() throws IOException, DataException, MissingDataException { public void build() throws IOException, DataException, MissingDataException {
Long now = NTP.getTime(); Long now = NTP.getTime();
if (now == null) { if (now == null) {
this.buildStartTimestamp = null;
throw new DataException("NTP time hasn't synced yet"); throw new DataException("NTP time hasn't synced yet");
} }
if (this.buildStartTimestamp == null) {
this.buildStartTimestamp = now; this.buildStartTimestamp = now;
}
ArbitraryDataReader arbitraryDataReader = ArbitraryDataReader arbitraryDataReader =
new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier); new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier);

5
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java

@ -42,8 +42,11 @@ public class ArbitraryDataBuildManager extends Thread {
try { try {
// Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread) // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
// This can be expanded to have multiple threads processing the build queue when needed // This can be expanded to have multiple threads processing the build queue when needed
ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1); int threadCount = 5;
ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(threadCount);
for (int i = 0; i < threadCount; i++) {
arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread()); arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
}
while (!isStopping) { while (!isStopping) {
// Nothing to do yet // Nothing to do yet

22
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java

@ -36,36 +36,40 @@ public class ArbitraryDataBuilderThread implements Runnable {
continue; continue;
} }
Map.Entry<String, ArbitraryDataBuildQueueItem> next = null; Long now = NTP.getTime();
if (now == null) {
continue;
}
ArbitraryDataBuildQueueItem queueItem = null;
// Find resources that are queued for building // Find resources that are queued for building
synchronized (buildManager.arbitraryDataBuildQueue) { synchronized (buildManager.arbitraryDataBuildQueue) {
next = buildManager.arbitraryDataBuildQueue Map.Entry<String, ArbitraryDataBuildQueueItem> next = buildManager.arbitraryDataBuildQueue
.entrySet().stream() .entrySet().stream()
.filter(e -> e.getValue().isQueued()) .filter(e -> e.getValue().isQueued())
.findFirst().orElse(null); .findFirst().orElse(null);
}
if (next == null) { if (next == null) {
continue; continue;
} }
Long now = NTP.getTime(); queueItem = next.getValue();
if (now == null) {
continue;
}
ArbitraryDataBuildQueueItem queueItem = next.getValue();
if (queueItem == null) { if (queueItem == null) {
this.removeFromQueue(queueItem); this.removeFromQueue(queueItem);
continue;
} }
// Ignore builds that have failed recently // Ignore builds that have failed recently
if (buildManager.isInFailedBuildsList(queueItem)) { if (buildManager.isInFailedBuildsList(queueItem)) {
this.removeFromQueue(queueItem);
continue; continue;
} }
// Set the start timestamp, to prevent other threads from building it at the same time
queueItem.prepareForBuild();
}
try { try {
// Perform the build // Perform the build

Loading…
Cancel
Save