Modified fetchArbitraryDataFileList() to support requesting only the missing hashes, but it is not yet used.

Once GetArbitraryDataFileListMessage is rolled out to the network we can uncomment this code. Needs testnet testing prior to that.

src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java

@@ -539,6 +539,50 @@ public class ArbitraryDataFile {
         return false;
     }
 
+    /**
+     * Retrieve a list of file hashes for this transaction that we do not hold locally
+     *
+     * @return a List of chunk hashes, or null if we are unable to determine what is missing
+     */
+    public List<byte[]> missingHashes() {
+        List<byte[]> missingHashes = new ArrayList<>();
+        try {
+            if (this.metadataHash == null) {
+                // We don't have any metadata so can't check if we have the chunks
+                // Even if this transaction has no chunks, we don't have the file either (already checked above)
+                return null;
+            }
+
+            if (this.metadataFile == null) {
+                this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature);
+            }
+
+            // If the metadata file doesn't exist, we can't check if we have the chunks
+            if (!metadataFile.getFilePath().toFile().exists()) {
+                return null;
+            }
+
+            if (this.metadata == null) {
+                this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath()));
+            }
+
+            // Read the metadata
+            List<byte[]> chunks = metadata.getChunks();
+            for (byte[] chunkHash : chunks) {
+                ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
+                if (!chunk.exists()) {
+                    missingHashes.add(chunkHash);
+                }
+            }
+
+            return missingHashes;
+
+        } catch (DataException e) {
+            // Something went wrong, so we can't make a sensible decision
+            return null;
+        }
+    }
+
     public boolean containsChunk(byte[] hash) {
         for (ArbitraryDataFileChunk chunk : this.chunks) {
             if (Arrays.equals(hash, chunk.getHash())) {

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java

@@ -236,9 +236,11 @@ public class ArbitraryDataFileListManager {
     }
 
 
-    // Lookup file lists by signature
+    // Lookup file lists by signature (and optionally hashes)
 
     public boolean fetchArbitraryDataFileList(ArbitraryTransactionData arbitraryTransactionData) {
+        byte[] digest = arbitraryTransactionData.getData();
+        byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
         byte[] signature = arbitraryTransactionData.getSignature();
         String signature58 = Base58.encode(signature);
 
@@ -261,10 +263,24 @@ public class ArbitraryDataFileListManager {
         this.addToSignatureRequests(signature58, true, false);
 
         List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
-        LOGGER.debug(String.format("Sending data file list request for signature %s to %d peers...", signature58, handshakedPeers.size()));
+        List<byte[]> missingHashes = null;
+
+//        // TODO: uncomment after GetArbitraryDataFileListMessage updates are deployed
+//        // Find hashes that we are missing
+//        try {
+//            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
+//            arbitraryDataFile.setMetadataHash(metadataHash);
+//            missingHashes = arbitraryDataFile.missingHashes();
+//        } catch (DataException e) {
+//            // Leave missingHashes as null, so that all hashes are requested
+//        }
+//        int hashCount = missingHashes != null ? missingHashes.size() : 0;
+
+        int hashCount = 0;
+        LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size()));
 
         // Build request
-        Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, now, 0);
+        Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, missingHashes, now, 0);
 
         // Save our request into requests map
         Triple<String, Peer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
@@ -313,12 +329,16 @@ public class ArbitraryDataFileListManager {
             return false;
         }
 
-        LOGGER.debug(String.format("Sending data file list request for signature %s to peer %s...", signature58, peer));
+        int hashCount = 0;
+        LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to peer %s...", signature58, hashCount, peer));
 
         // Build request
         // Use a time in the past, so that the recipient peer doesn't try and relay it
+        // Also, set hashes to null since it's easier to request all hashes than it is to determine which ones we need
+        // This could be optimized in the future
         long timestamp = now - 60000L;
-        Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, timestamp, 0);
+        List<byte[]> hashes = null;
+        Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, timestamp, 0);
 
         // Save our request into requests map
         Triple<String, Peer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());