From 16bcba6e2e90cbc4deba8968693574cb5370134b Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Tue, 2 Nov 2021 19:32:48 +0000
Subject: [PATCH] When accessing a website or other data resource, request the
 chunks if we don't already have them.

This causes the build to fail on the first pass due to missing chunks, however it now fails with a message indicating that it should be retried shortly. The website loader is already set up in such a way that it will be automatically retried, during which time the loading screen is shown.

Also added code to remove the resource from the "failed builds list" once the chunks arrive, so that it is able to be rebuilt sooner than the FAILURE_TIMEOUT (currently 5 minutes).
---
 .../qortal/arbitrary/ArbitraryDataReader.java | 11 +++++++---
 .../arbitrary/ArbitraryDataManager.java       | 22 ++++++++++++++++---
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
index 4ff4b26e..a4820e37 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
@@ -282,9 +282,14 @@ public class ArbitraryDataReader {
         ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
         if (!arbitraryDataFile.exists()) {
             if (!arbitraryDataFile.allChunksExist(chunkHashes)) {
-                // TODO: fetch them?
-                LOGGER.info(String.format("Missing chunks for file %s", arbitraryDataFile));
-                throw new IllegalStateException(String.format("Missing chunks for file %s", arbitraryDataFile));
+
+                // Ask the arbitrary data manager to fetch data for this transaction
+                ArbitraryDataManager.getInstance().fetchDataForSignature(transactionData.getSignature());
+
+                // Fail the build, as it will be retried later once the chunks arrive
+                String response = String.format("Missing chunks for file %s have been requested. Please try again once they have been received.", arbitraryDataFile);
+                LOGGER.info(response);
+                throw new IllegalStateException(response);
             }
             // We have all the chunks but not the complete file, so join them
             arbitraryDataFile.addChunkHashes(chunkHashes);
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
index 97cf7a18..e5c8c1b1 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
@@ -199,7 +199,7 @@ public class ArbitraryDataManager extends Thread {
 
 					// Ask our connected peers if they have files for this signature
 					// This process automatically then fetches the files themselves if a peer is found
-					fetchArbitraryDataFileList(signature);
+					fetchDataForSignature(signature);
 
 				} catch (DataException e) {
 					LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
@@ -238,7 +238,11 @@ public class ArbitraryDataManager extends Thread {
 		}
 	}
 
-	private boolean fetchArbitraryDataFileList(byte[] signature) throws InterruptedException {
+	public boolean fetchDataForSignature(byte[] signature) {
+		return this.fetchArbitraryDataFileList(signature);
+	}
+
+	private boolean fetchArbitraryDataFileList(byte[] signature) {
 		LOGGER.info(String.format("Sending data file list request for signature %s", Base58.encode(signature)));
 		// Build request
 		Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature);
@@ -264,7 +268,11 @@ public class ArbitraryDataManager extends Thread {
 		final long singleWait = 100;
 		long totalWait = 0;
 		while (totalWait < ARBITRARY_REQUEST_TIMEOUT) {
-			Thread.sleep(singleWait);
+			try {
+				Thread.sleep(singleWait);
+			} catch (InterruptedException e) {
+				break;
+			}
 
 			requestEntry = arbitraryDataFileListRequests.get(id);
 			if (requestEntry == null)
@@ -595,9 +603,17 @@ public class ArbitraryDataManager extends Thread {
 				// data cache so that it is rebuilt the next time we serve it
 				if (arbitraryTransactionData.getName() != null) {
 					String resourceId = arbitraryTransactionData.getName().toLowerCase();
+					LOGGER.info("We have all data for transaction {}", Base58.encode(transactionData.getSignature()));
+					LOGGER.info("Clearing cache for name {}...", arbitraryTransactionData.getName());
+
 					if (this.arbitraryDataCachedResources.containsKey(resourceId)) {
 						this.arbitraryDataCachedResources.remove(resourceId);
 					}
+
+					// Also remove from the failed builds queue in case it previously failed due to missing chunks
+					if (this.arbitraryDataFailedBuilds.containsKey(resourceId)) {
+						this.arbitraryDataFailedBuilds.remove(resourceId);
+					}
 				}
 
 				// We also need to broadcast to the network that we are now hosting files for this transaction