Browse Source

Use 5 builder threads, so that one slow resource (e.g. a thumbnail) doesn't hold up the other queued build items.

This can be replaced with a task-based approach longer term.
online-level-zero-accounts-api-call
CalDescent 3 years ago
parent
commit
fc82f0b622
  1. 9
      src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java
  2. 7
      src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java
  3. 40
      src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java

9
src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java

@ -27,13 +27,20 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
this.creationTimestamp = NTP.getTime(); this.creationTimestamp = NTP.getTime();
} }
public void prepareForBuild() {
this.buildStartTimestamp = NTP.getTime();
}
public void build() throws IOException, DataException, MissingDataException { public void build() throws IOException, DataException, MissingDataException {
Long now = NTP.getTime(); Long now = NTP.getTime();
if (now == null) { if (now == null) {
this.buildStartTimestamp = null;
throw new DataException("NTP time hasn't synced yet"); throw new DataException("NTP time hasn't synced yet");
} }
this.buildStartTimestamp = now; if (this.buildStartTimestamp == null) {
this.buildStartTimestamp = now;
}
ArbitraryDataReader arbitraryDataReader = ArbitraryDataReader arbitraryDataReader =
new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier); new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier);

7
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java

@ -42,8 +42,11 @@ public class ArbitraryDataBuildManager extends Thread {
try { try {
// Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread) // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
// This can be expanded to have multiple threads processing the build queue when needed // This can be expanded to have multiple threads processing the build queue when needed
ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1); int threadCount = 5;
arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread()); ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(threadCount);
for (int i = 0; i < threadCount; i++) {
arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
}
while (!isStopping) { while (!isStopping) {
// Nothing to do yet // Nothing to do yet

40
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java

@ -36,37 +36,41 @@ public class ArbitraryDataBuilderThread implements Runnable {
continue; continue;
} }
Map.Entry<String, ArbitraryDataBuildQueueItem> next = null; Long now = NTP.getTime();
if (now == null) {
continue;
}
ArbitraryDataBuildQueueItem queueItem = null;
// Find resources that are queued for building // Find resources that are queued for building
synchronized (buildManager.arbitraryDataBuildQueue) { synchronized (buildManager.arbitraryDataBuildQueue) {
next = buildManager.arbitraryDataBuildQueue Map.Entry<String, ArbitraryDataBuildQueueItem> next = buildManager.arbitraryDataBuildQueue
.entrySet().stream() .entrySet().stream()
.filter(e -> e.getValue().isQueued()) .filter(e -> e.getValue().isQueued())
.findFirst().orElse(null); .findFirst().orElse(null);
}
if (next == null) { if (next == null) {
continue; continue;
} }
Long now = NTP.getTime(); queueItem = next.getValue();
if (now == null) {
continue;
}
ArbitraryDataBuildQueueItem queueItem = next.getValue(); if (queueItem == null) {
this.removeFromQueue(queueItem);
continue;
}
if (queueItem == null) { // Ignore builds that have failed recently
this.removeFromQueue(queueItem); if (buildManager.isInFailedBuildsList(queueItem)) {
} this.removeFromQueue(queueItem);
continue;
}
// Ignore builds that have failed recently // Set the start timestamp, to prevent other threads from building it at the same time
if (buildManager.isInFailedBuildsList(queueItem)) { queueItem.prepareForBuild();
continue;
} }
try { try {
// Perform the build // Perform the build
LOGGER.info("Building {}...", queueItem); LOGGER.info("Building {}...", queueItem);

Loading…
Cancel
Save