From 077165b80731e6d714c5fbabf9a7a401b6563c71 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 3 Feb 2022 20:01:44 +0000 Subject: [PATCH] Modified fetchArbitraryDataFileList() to support requesting only the missing hashes, but it is not yet used. Once GetArbitraryDataFileListMessage is rolled out to the network we can uncomment this code. Needs testnet testing prior to that. --- .../qortal/arbitrary/ArbitraryDataFile.java | 44 +++++++++++++++++++ .../ArbitraryDataFileListManager.java | 30 ++++++++++--- 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 1307eab7..2d7346ea 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -539,6 +539,50 @@ public class ArbitraryDataFile { return false; } + /** + * Retrieve a list of file hashes for this transaction that we do not hold locally + * + * @return a List of chunk hashes, or null if we are unable to determine what is missing + */ + public List missingHashes() { + List missingHashes = new ArrayList<>(); + try { + if (this.metadataHash == null) { + // We don't have any metadata so can't check if we have the chunks + // Even if this transaction has no chunks, we don't have the file either (already checked above) + return null; + } + + if (this.metadataFile == null) { + this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature); + } + + // If the metadata file doesn't exist, we can't check if we have the chunks + if (!metadataFile.getFilePath().toFile().exists()) { + return null; + } + + if (this.metadata == null) { + this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath())); + } + + // Read the metadata + List chunks = metadata.getChunks(); + for (byte[] chunkHash : chunks) { + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature); + if (!chunk.exists()) { + missingHashes.add(chunkHash); + } + } + + return missingHashes; + + } catch (DataException e) { + // Something went wrong, so we can't make a sensible decision + return null; + } + } + public boolean containsChunk(byte[] hash) { for (ArbitraryDataFileChunk chunk : this.chunks) { if (Arrays.equals(hash, chunk.getHash())) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index 3147f9cb..6337fc7c 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -236,9 +236,11 @@ public class ArbitraryDataFileListManager { } - // Lookup file lists by signature + // Lookup file lists by signature (and optionally hashes) public boolean fetchArbitraryDataFileList(ArbitraryTransactionData arbitraryTransactionData) { + byte[] digest = arbitraryTransactionData.getData(); + byte[] metadataHash = arbitraryTransactionData.getMetadataHash(); byte[] signature = arbitraryTransactionData.getSignature(); String signature58 = Base58.encode(signature); @@ -261,10 +263,24 @@ public class ArbitraryDataFileListManager { this.addToSignatureRequests(signature58, true, false); List handshakedPeers = Network.getInstance().getHandshakedPeers(); - LOGGER.debug(String.format("Sending data file list request for signature %s to %d peers...", signature58, handshakedPeers.size())); + List missingHashes = null; + +// // TODO: uncomment after GetArbitraryDataFileListMessage updates are deployed +// // Find hashes that we are missing +// try { +// ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); +// arbitraryDataFile.setMetadataHash(metadataHash); +// missingHashes = arbitraryDataFile.missingHashes(); +// } catch (DataException e) { +// // Leave missingHashes as null, so that all hashes are requested +// } +// int hashCount = missingHashes != null ? missingHashes.size() : 0; + + int hashCount = 0; + LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size())); // Build request - Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, now, 0); + Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, missingHashes, now, 0); // Save our request into requests map Triple requestEntry = new Triple<>(signature58, null, NTP.getTime()); @@ -313,12 +329,16 @@ public class ArbitraryDataFileListManager { return false; } - LOGGER.debug(String.format("Sending data file list request for signature %s to peer %s...", signature58, peer)); + int hashCount = 0; + LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to peer %s...", signature58, hashCount, peer)); // Build request // Use a time in the past, so that the recipient peer doesn't try and relay it + // Also, set hashes to null since it's easier to request all hashes than it is to determine which ones we need + // This could be optimized in the future long timestamp = now - 60000L; - Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, timestamp, 0); + List hashes = null; + Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, timestamp, 0); // Save our request into requests map Triple requestEntry = new Triple<>(signature58, null, NTP.getTime());