Browse Source

Use 5 builder threads, so that one slow resource (e.g. a thumbnail) doesn't hold up the other queued build items.

This can be replaced with a task-based approach longer term.
online-level-zero-accounts-api-call
CalDescent 3 years ago
parent
commit
fc82f0b622
  1. 9
      src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java
  2. 7
      src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java
  3. 40
      src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java

9
src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java

@ -27,13 +27,20 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
this.creationTimestamp = NTP.getTime();
}
public void prepareForBuild() {
this.buildStartTimestamp = NTP.getTime();
}
public void build() throws IOException, DataException, MissingDataException {
Long now = NTP.getTime();
if (now == null) {
this.buildStartTimestamp = null;
throw new DataException("NTP time hasn't synced yet");
}
this.buildStartTimestamp = now;
if (this.buildStartTimestamp == null) {
this.buildStartTimestamp = now;
}
ArbitraryDataReader arbitraryDataReader =
new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier);

7
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java

@ -42,8 +42,11 @@ public class ArbitraryDataBuildManager extends Thread {
try {
// Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
// This can be expanded to have multiple threads processing the build queue when needed
ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1);
arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
int threadCount = 5;
ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(threadCount);
for (int i = 0; i < threadCount; i++) {
arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
}
while (!isStopping) {
// Nothing to do yet

40
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java

@ -36,37 +36,41 @@ public class ArbitraryDataBuilderThread implements Runnable {
continue;
}
Map.Entry<String, ArbitraryDataBuildQueueItem> next = null;
Long now = NTP.getTime();
if (now == null) {
continue;
}
ArbitraryDataBuildQueueItem queueItem = null;
// Find resources that are queued for building
synchronized (buildManager.arbitraryDataBuildQueue) {
next = buildManager.arbitraryDataBuildQueue
Map.Entry<String, ArbitraryDataBuildQueueItem> next = buildManager.arbitraryDataBuildQueue
.entrySet().stream()
.filter(e -> e.getValue().isQueued())
.findFirst().orElse(null);
}
if (next == null) {
continue;
}
if (next == null) {
continue;
}
Long now = NTP.getTime();
if (now == null) {
continue;
}
queueItem = next.getValue();
ArbitraryDataBuildQueueItem queueItem = next.getValue();
if (queueItem == null) {
this.removeFromQueue(queueItem);
continue;
}
if (queueItem == null) {
this.removeFromQueue(queueItem);
}
// Ignore builds that have failed recently
if (buildManager.isInFailedBuildsList(queueItem)) {
this.removeFromQueue(queueItem);
continue;
}
// Ignore builds that have failed recently
if (buildManager.isInFailedBuildsList(queueItem)) {
continue;
// Set the start timestamp, to prevent other threads from building it at the same time
queueItem.prepareForBuild();
}
try {
// Perform the build
LOGGER.info("Building {}...", queueItem);

Loading…
Cancel
Save