diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 656bf949..82ac1f30 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -30,6 +30,8 @@ import org.qortal.api.*; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; import org.qortal.arbitrary.ArbitraryDataReader; import org.qortal.arbitrary.ArbitraryDataTransactionBuilder; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.controller.Controller; import org.qortal.data.account.AccountData; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.ArbitraryTransactionData; @@ -259,7 +261,16 @@ public class ArbitraryResource { Service service = Service.valueOf(serviceString); ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service); try { - arbitraryDataReader.loadSynchronously(rebuild); + + // Loop until we have data + while (!Controller.isStopping()) { + try { + arbitraryDataReader.loadSynchronously(rebuild); + break; + } catch (MissingDataException e) { + continue; + } + } // TODO: limit file size that can be read into memory java.nio.file.Path path = Paths.get(arbitraryDataReader.getFilePath().toString(), filepath); diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 5c8951a6..50edac3e 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -31,6 +31,7 @@ import org.qortal.api.ApiExceptionFactory; import org.qortal.api.HTMLParser; import org.qortal.api.Security; import org.qortal.arbitrary.ArbitraryDataTransactionBuilder; +import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.repository.DataException; @@ -145,7 +146,7 @@ public class WebsiteResource { ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), name, service, method, compression); try { arbitraryDataWriter.save(); - } catch (IOException | DataException | InterruptedException e) { + } catch (IOException | DataException | InterruptedException | MissingDataException e) { LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (RuntimeException e) { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java index c84e36a9..fb3e534c 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -1,5 +1,6 @@ package org.qortal.arbitrary; +import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.repository.DataException; @@ -30,7 +31,7 @@ public class ArbitraryDataBuildQueueItem { this.creationTimestamp = NTP.getTime(); } - public void build() throws IOException, DataException { + public void build() throws IOException, DataException, MissingDataException { Long now = NTP.getTime(); if (now == null) { throw new IllegalStateException("NTP time hasn't synced yet"); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index 6183c401..b9dd2652 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -2,6 +2,7 @@ package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.Method; @@ -40,7 +41,7 @@ public class ArbitraryDataBuilder { this.paths = new ArrayList<>(); } - public void build() throws DataException, IOException { + public void build() throws DataException, IOException, MissingDataException { this.fetchTransactions(); this.validateTransactions(); this.processTransactions(); @@ -104,17 +105,37 @@ public class ArbitraryDataBuilder { } } - private void processTransactions() throws IOException, DataException { + private void processTransactions() throws IOException, DataException, MissingDataException { List transactionDataList = new ArrayList<>(this.transactions); + int count = 0; for (ArbitraryTransactionData transactionData : transactionDataList) { LOGGER.trace("Found arbitrary transaction {}", Base58.encode(transactionData.getSignature())); + count++; // Build the data file, overwriting anything that was previously there String sig58 = Base58.encode(transactionData.getSignature()); ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service); arbitraryDataReader.setTransactionData(transactionData); - arbitraryDataReader.loadSynchronously(true); + boolean hasMissingData = false; + try { + arbitraryDataReader.loadSynchronously(true); + } + catch (MissingDataException e) { + hasMissingData = true; + } + + // Handle missing data + if (hasMissingData) { + if (count == transactionDataList.size()) { + // This is the final transaction in the list, so we need to fail + throw new MissingDataException("Requesting missing files. Please wait and try again."); + } + // There are more transactions, so we should process them to give them the opportunity to request data + continue; + } + + // By this point we should have all data needed to build the layers Path path = arbitraryDataReader.getFilePath(); if (path == null) { throw new IllegalStateException(String.format("Null path when building data from transaction %s", sig58)); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 39fc7b7f..e95801d0 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.crypto.AES; @@ -33,6 +34,7 @@ import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.util.Arrays; +import java.util.MissingResourceException; public class ArbitraryDataReader { @@ -113,7 +115,7 @@ public class ArbitraryDataReader { * @throws IOException * @throws DataException */ - public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException { + public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException, MissingDataException { try { ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite, this.resourceId, this.resourceIdType, this.service); @@ -197,7 +199,7 @@ public class ArbitraryDataReader { } } - private void fetch() throws IllegalStateException, IOException, DataException { + private void fetch() throws IllegalStateException, IOException, DataException, MissingDataException { switch (resourceIdType) { case FILE_HASH: @@ -228,7 +230,7 @@ public class ArbitraryDataReader { this.filePath = arbitraryDataFile.getFilePath(); } - private void fetchFromName() throws IllegalStateException, IOException, DataException { + private void fetchFromName() throws IllegalStateException, IOException, DataException, MissingDataException { try { // Build the existing state using past transactions @@ -250,7 +252,7 @@ public class ArbitraryDataReader { } } - private void fetchFromSignature() throws IllegalStateException, IOException, DataException { + private void fetchFromSignature() throws IllegalStateException, IOException, DataException, MissingDataException { // Load the full transaction data from the database so we can access the file hashes ArbitraryTransactionData transactionData; @@ -264,7 +266,7 @@ public class ArbitraryDataReader { this.fetchFromTransactionData(transactionData); } - private void fetchFromTransactionData(ArbitraryTransactionData transactionData) throws IllegalStateException, IOException, DataException { + private void fetchFromTransactionData(ArbitraryTransactionData transactionData) throws IllegalStateException, IOException, MissingDataException { if (!(transactionData instanceof ArbitraryTransactionData)) { throw new IllegalStateException(String.format("Transaction data not found for signature %s", this.resourceId)); } @@ -287,10 +289,10 @@ public class ArbitraryDataReader { // Ask the arbitrary data manager to fetch data for this transaction ArbitraryDataManager.getInstance().fetchDataForSignature(transactionData.getSignature()); - // Fail the build, as it will be retried later once the chunks arrive - String response = String.format("Missing chunks for file %s have been requested. Please try again once they have been received.", arbitraryDataFile); - LOGGER.info(response); - throw new IllegalStateException(response); + // Throw a missing data exception, which allows subsequent layers to fetch data + String message = String.format("Requested missing data for file %s", arbitraryDataFile); + LOGGER.info(message); + throw new MissingDataException(message); } // We have all the chunks but not the complete file, so join them arbitraryDataFile.addChunkHashes(chunkHashes); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java index 0b79e96f..83cc6022 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java @@ -2,6 +2,7 @@ package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; import org.qortal.data.PaymentData; @@ -68,7 +69,7 @@ public class ArbitraryDataTransactionBuilder { ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, method, compression); try { arbitraryDataWriter.save(); - } catch (IOException | DataException | InterruptedException | RuntimeException e) { + } catch (IOException | DataException | InterruptedException | RuntimeException | MissingDataException e) { LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw new DataException(String.format("Unable to create arbitrary data file: %s", e.getMessage())); } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index e9c9d454..faaa548d 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -3,6 +3,7 @@ package org.qortal.arbitrary; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.crypto.Crypto; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.crypto.AES; @@ -52,7 +53,7 @@ public class ArbitraryDataWriter { this.compression = compression; } - public void save() throws IllegalStateException, IOException, DataException, InterruptedException { + public void save() throws IllegalStateException, IOException, DataException, InterruptedException, MissingDataException { try { this.preExecute(); this.process(); @@ -94,7 +95,7 @@ public class ArbitraryDataWriter { this.workingPath = tempDir; } - private void process() throws DataException, IOException { + private void process() throws DataException, IOException, MissingDataException { switch (this.method) { case PUT: @@ -110,7 +111,7 @@ public class ArbitraryDataWriter { } } - private void processPatch() throws DataException, IOException { + private void processPatch() throws DataException, IOException, MissingDataException { // Build the existing state using past transactions ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.name, this.service); diff --git a/src/main/java/org/qortal/arbitrary/exception/MissingDataException.java b/src/main/java/org/qortal/arbitrary/exception/MissingDataException.java new file mode 100644 index 00000000..63f617c0 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/exception/MissingDataException.java @@ -0,0 +1,20 @@ +package org.qortal.arbitrary.exception; + +public class MissingDataException extends Exception { + + public MissingDataException() { + } + + public MissingDataException(String message) { + super(message); + } + + public MissingDataException(String message, Throwable cause) { + super(message, cause); + } + + public MissingDataException(Throwable cause) { + super(cause); + } + +} diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java index 64866f7f..d7d5a12d 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java @@ -3,6 +3,7 @@ package org.qortal.controller.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; +import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.controller.Controller; import org.qortal.repository.DataException; import org.qortal.utils.NTP; @@ -69,6 +70,12 @@ public class ArbitraryDataBuilderThread implements Runnable { this.removeFromQueue(resourceId); LOGGER.info("Finished building {}", queueItem); + } catch (MissingDataException e) { + LOGGER.info("Missing data for {}: {}", queueItem, e.getMessage()); + queueItem.setFailed(true); + this.removeFromQueue(resourceId); + // Don't add to the failed builds list, as we may want to retry sooner + } catch (IOException | DataException | RuntimeException e) { LOGGER.info("Error building {}: {}", queueItem, e.getMessage()); // Something went wrong - so remove it from the queue, and add to failed builds list diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java index 6b954da3..0f3c3f50 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java @@ -7,6 +7,7 @@ import org.qortal.arbitrary.ArbitraryDataDigest; import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.arbitrary.ArbitraryDataReader; import org.qortal.arbitrary.ArbitraryDataTransactionBuilder; +import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; @@ -37,7 +38,7 @@ public class ArbitraryDataTests extends Common { } @Test - public void testCombineMultipleLayers() throws DataException, IOException { + public void testCombineMultipleLayers() throws DataException, IOException, MissingDataException { try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String publicKey58 = Base58.encode(alice.getPublicKey()); @@ -159,7 +160,7 @@ public class ArbitraryDataTests extends Common { } @Test - public void testUpdateResource() throws DataException, IOException { + public void testUpdateResource() throws DataException, IOException, MissingDataException { try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String publicKey58 = Base58.encode(alice.getPublicKey());