diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java index 2cabd3ec..75769b27 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -12,14 +12,22 @@ public class ArbitraryDataBuildQueueItem { private String resourceId; private ResourceIdType resourceIdType; private Service service; + private Long creationTimestamp = null; private Long buildStartTimestamp = null; + private Long buildEndTimestamp = null; + private boolean failed = false; - private static long BUILD_TIMEOUT = 60*1000L; // 60 seconds + /* The maximum amount of time to spend on a single build */ + // TODO: interrupt an in-progress build + public static long BUILD_TIMEOUT = 60*1000L; // 60 seconds + /* The amount of time to remember that a build has failed, to avoid retries */ + public static long FAILURE_TIMEOUT = 1*60*1000L; // 5 minutes public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service) { this.resourceId = resourceId; this.resourceIdType = resourceIdType; this.service = service; + this.creationTimestamp = NTP.getTime(); } public void build() throws IOException, DataException { @@ -34,7 +42,11 @@ public class ArbitraryDataBuildQueueItem { // We do not want to overwrite the existing cache, as this will be invalidated // automatically if new data has arrived - arbitraryDataReader.loadSynchronously(false); + try { + arbitraryDataReader.loadSynchronously(false); + } finally { + this.buildEndTimestamp = NTP.getTime(); + } } public boolean isBuilding() { @@ -46,10 +58,17 @@ public class ArbitraryDataBuildQueueItem { } public boolean hasReachedBuildTimeout(Long now) { + if (now == null || this.creationTimestamp == null) { + return true; + } + return now - this.creationTimestamp > BUILD_TIMEOUT; + } + + public boolean hasReachedFailureTimeout(Long now) { if (now == null || this.buildStartTimestamp == null) { return true; } - return now - this.buildStartTimestamp > BUILD_TIMEOUT; + return now - this.buildStartTimestamp > FAILURE_TIMEOUT; } @@ -61,6 +80,11 @@ public class ArbitraryDataBuildQueueItem { return this.buildStartTimestamp; } + public void setFailed(boolean failed) { + this.failed = failed; + } + + @Override public String toString() { return String.format("%s %s", this.service, this.resourceId); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java index 9fa383c0..52eaf5b5 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -36,7 +36,9 @@ public class ArbitraryDataBuildManager implements Runnable { // Find resources that are queued for building Map.Entry next = arbitraryDataManager.arbitraryDataBuildQueue - .entrySet().stream().filter(e -> e.getValue().isQueued()).findFirst().get(); + .entrySet().stream() + .filter(e -> e.getValue().isQueued()) + .findFirst().get(); if (next == null) { continue; @@ -49,10 +51,17 @@ public class ArbitraryDataBuildManager implements Runnable { String resourceId = next.getKey(); ArbitraryDataBuildQueueItem queueItem = next.getValue(); - if (queueItem == null || queueItem.hasReachedBuildTimeout(now)) { + + if (queueItem == null) { this.removeFromQueue(resourceId); } + // Ignore builds that have failed recently + if (ArbitraryDataManager.getInstance().isInFailedBuildsList(queueItem)) { + continue; + } + + try { // Perform the build LOGGER.info("Building {}...", queueItem); @@ -62,8 +71,9 @@ public class ArbitraryDataBuildManager implements Runnable { } catch (IOException | DataException e) { LOGGER.info("Error building {}: {}", queueItem, e.getMessage()); - // Something went wrong - so remove it from the queue - // TODO: we may want to keep track of this in a "cooloff" list to prevent frequent re-attempts + // Something went wrong - so remove it from the queue, and add to failed builds list + queueItem.setFailed(true); + ArbitraryDataManager.getInstance().addToFailedBuildsList(queueItem); this.removeFromQueue(resourceId); } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 4a99f55e..be43b20b 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -79,6 +79,11 @@ public class ArbitraryDataManager extends Thread { */ public Map arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>()); + /** + * Map to keep track of failed arbitrary transaction builds. + */ + public Map arbitraryDataFailedBuilds = Collections.synchronizedMap(new HashMap<>()); + private ArbitraryDataManager() { } @@ -222,12 +227,23 @@ public class ArbitraryDataManager extends Thread { return arbitraryDataFileMessage.getArbitraryDataFile(); } - public void cleanupRequestCache(long now) { + public void cleanupRequestCache(Long now) { + if (now == null) { + return; + } final long requestMinimumTimestamp = now - ARBITRARY_REQUEST_TIMEOUT; - arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); // TODO: fix NPE + arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() < requestMinimumTimestamp); } + public void cleanupQueues(Long now) { + if (now == null) { + return; + } + arbitraryDataBuildQueue.entrySet().removeIf(entry -> entry.getValue().hasReachedBuildTimeout(now)); + arbitraryDataFailedBuilds.entrySet().removeIf(entry -> entry.getValue().hasReachedFailureTimeout(now)); + } + // Arbitrary data resource cache public boolean isResourceCached(String resourceId) { @@ -272,6 +288,7 @@ public class ArbitraryDataManager extends Thread { } // Build queue + public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) { String resourceId = queueItem.getResourceId(); if (resourceId == null) { @@ -287,6 +304,11 @@ public class ArbitraryDataManager extends Thread { return false; } + // Don't add builds that have failed recently + if (this.isInFailedBuildsList(queueItem)) { + return false; + } + if (this.arbitraryDataBuildQueue.put(resourceId, queueItem) != null) { // Already in queue return true; @@ -318,6 +340,54 @@ public class ArbitraryDataManager extends Thread { } + // Failed builds + + public boolean addToFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) { + String resourceId = queueItem.getResourceId(); + if (resourceId == null) { + return false; + } + + if (this.arbitraryDataFailedBuilds == null) { + return false; + } + + if (NTP.getTime() == null) { + // Can't use queues until we have synced the time + return false; + } + + if (this.arbitraryDataFailedBuilds.put(resourceId, queueItem) != null) { + // Already in list + return true; + } + + LOGGER.info("Added {} to failed builds list", resourceId); + + // Added to queue + return true; + } + + public boolean isInFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) { + String resourceId = queueItem.getResourceId(); + if (resourceId == null) { + return false; + } + + if (this.arbitraryDataFailedBuilds == null) { + return false; + } + + if (this.arbitraryDataFailedBuilds.containsKey(resourceId)) { + // Already in list + return true; + } + + // Not in list + return false; + } + + // Network handlers public void onNetworkGetArbitraryDataMessage(Peer peer, Message message) {