diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index a062471a..1f67da9c 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -291,7 +291,20 @@ public class ArbitraryDataFileListManager { return true; } - + public void deleteFileListRequestsForSignature(byte[] signature) { + String signature58 = Base58.encode(signature); + for (Iterator>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) { + Map.Entry> entry = it.next(); + if (entry == null || entry.getKey() == null || entry.getValue() != null) { + continue; + } + if (Objects.equals(entry.getValue().getA(), signature58)) { + // Update requests map to reflect that we've received all chunks + Triple newEntry = new Triple<>(null, null, entry.getValue().getC()); + arbitraryDataFileListRequests.put(entry.getKey(), newEntry); + } + } + } // Network handlers @@ -304,7 +317,7 @@ public class ArbitraryDataFileListManager { ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size()); - // Do we have a pending request for this data? // TODO: might we want to relay all of them anyway? + // Do we have a pending request for this data? Triple request = arbitraryDataFileListRequests.get(message.getId()); if (request == null || request.getA() == null) { return; @@ -350,10 +363,6 @@ public class ArbitraryDataFileListManager { // } // } - // Update requests map to reflect that we've received it - Triple newEntry = new Triple<>(null, null, request.getC()); - arbitraryDataFileListRequests.put(message.getId(), newEntry); - if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) { // Go and fetch the actual data, since this isn't a relay request arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, hashes); @@ -412,6 +421,7 @@ public class ArbitraryDataFileListManager { List hashes = new ArrayList<>(); ArbitraryTransactionData transactionData = null; + boolean allChunksExist = false; try (final Repository repository = RepositoryManager.getRepository()) { @@ -435,6 +445,9 @@ public class ArbitraryDataFileListManager { hashes.add(arbitraryDataFile.getMetadataHash()); } + // Check if we have all the chunks (and the metadata file) + allChunksExist = arbitraryDataFile.allChunksExist(); + for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) { if (chunk.exists()) { hashes.add(chunk.getHash()); @@ -459,49 +472,59 @@ public class ArbitraryDataFileListManager { // We should only respond if we have at least one hash if (hashes.size() > 0) { - // Update requests map to reflect that we've sent it - newEntry = new Triple<>(signature58, null, now); - arbitraryDataFileListRequests.put(message.getId(), newEntry); + // We have all the chunks, so update requests map to reflect that we've sent it + // There is no need to keep track of the request, as we can serve all the chunks + if (allChunksExist) { + newEntry = new Triple<>(null, null, now); + arbitraryDataFileListRequests.put(message.getId(), newEntry); + } ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); arbitraryDataFileListMessage.setId(message.getId()); if (!peer.sendMessage(arbitraryDataFileListMessage)) { LOGGER.debug("Couldn't send list of hashes"); peer.disconnect("failed to send list of hashes"); + return; } LOGGER.debug("Sent list of hashes (count: {})", hashes.size()); + if (allChunksExist) { + // Nothing left to do, so return to prevent any unnecessary forwarding from occurring + LOGGER.debug("No need for any forwarding because file list request is fully served"); + return; + } + } - else { - boolean isBlocked = (transactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName())); - if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { - // In relay mode - so ask our other peers if they have it - - long requestTime = getArbitraryDataFileListMessage.getRequestTime(); - int requestHops = getArbitraryDataFileListMessage.getRequestHops(); - getArbitraryDataFileListMessage.setRequestHops(++requestHops); - long totalRequestTime = now - requestTime; - - if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { - // Relay request hasn't timed out yet, so can potentially be rebroadcast - if (requestHops < RELAY_REQUEST_MAX_HOPS) { - // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast - - LOGGER.info("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); - Network.getInstance().broadcast( - broadcastPeer -> broadcastPeer == peer || - Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) - ? null : getArbitraryDataFileListMessage); - } - else { - // This relay request has reached the maximum number of allowed hops - } + // We may need to forward this request on + boolean isBlocked = (transactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName())); + if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { + // In relay mode - so ask our other peers if they have it + + long requestTime = getArbitraryDataFileListMessage.getRequestTime(); + int requestHops = getArbitraryDataFileListMessage.getRequestHops(); + getArbitraryDataFileListMessage.setRequestHops(++requestHops); + long totalRequestTime = now - requestTime; + + if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { + // Relay request hasn't timed out yet, so can potentially be rebroadcast + if (requestHops < RELAY_REQUEST_MAX_HOPS) { + // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast + + LOGGER.info("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); + Network.getInstance().broadcast( + broadcastPeer -> broadcastPeer == peer || + Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) + ? null : getArbitraryDataFileListMessage); + } else { - // This relay request has timed out + // This relay request has reached the maximum number of allowed hops } } + else { + // This relay request has timed out + } } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 2b0b8143..0f58a6b0 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -15,6 +15,7 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; +import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.Base58; import org.qortal.utils.NTP; import org.qortal.utils.Triple; @@ -187,6 +188,9 @@ public class ArbitraryDataFileManager { arbitraryDataFileRequests.remove(hash58); LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58)); + // We may need to remove the file list request, if we have all the files for this transaction + this.handleFileListRequests(signature); + if (message == null || message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) { return null; } @@ -209,6 +213,26 @@ public class ArbitraryDataFileManager { return arbitraryDataFileMessage; } + private void handleFileListRequests(byte[] signature) { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Fetch the transaction data + ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); + if (arbitraryTransactionData == null) { + return; + } + + boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData); + + if (allChunksExist) { + // Update requests map to reflect that we've received all chunks + ArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature); + } + + } catch (DataException e) { + LOGGER.debug("Unable to handle file list requests: {}", e.getMessage()); + } + } public void handleArbitraryDataFileForwarding(Peer requestingPeer, Message message, Message originalMessage) { // Return if there is no originally requesting peer to forward to