From 941080c39527f0207c3c70a7b02d8dd1edb82d8f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 4 Mar 2022 13:33:17 +0000 Subject: [PATCH] Rework of arbitraryDataFileHashResponses to use a list rather than a map (limited to 1000) items. Sort the list by routes with the lowest number of peer hops first, to try and prioritize those which are easiest and quickest to reach. --- .../ArbitraryDataFileListManager.java | 23 +++-- .../arbitrary/ArbitraryDataFileManager.java | 12 +-- .../ArbitraryDataFileRequestThread.java | 85 ++++++++++--------- .../ArbitraryFileListResponseInfo.java | 11 +++ 4 files changed, 78 insertions(+), 53 deletions(-) create mode 100644 src/main/java/org/qortal/data/arbitrary/ArbitraryFileListResponseInfo.java diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index 956280e3..dd6df568 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.arbitrary.ArbitraryDataFileChunk; import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo; import org.qortal.data.arbitrary.ArbitraryRelayInfo; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; @@ -23,6 +24,8 @@ import org.qortal.utils.Triple; import java.util.*; +import static org.qortal.controller.arbitrary.ArbitraryDataFileManager.MAX_FILE_HASH_RESPONSES; + public class ArbitraryDataFileListManager { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileListManager.class); @@ -458,12 +461,20 @@ public class ArbitraryDataFileListManager { // } if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) { - // Keep track of the hashes this peer reports to have access to - Long now = NTP.getTime(); - for (byte[] hash : hashes) { - String hash58 = Base58.encode(hash); - String sig58 = Base58.encode(signature); - ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.put(hash58, new Triple<>(peer, sig58, now)); + if (ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.size() < MAX_FILE_HASH_RESPONSES) { + // Keep track of the hashes this peer reports to have access to + Long now = NTP.getTime(); + for (byte[] hash : hashes) { + String hash58 = Base58.encode(hash); + + // Treat null request hops as 100, so that they are able to be sorted (and put to the end of the list) + int requestHops = arbitraryDataFileListMessage.getRequestHops() != null ? arbitraryDataFileListMessage.getRequestHops() : 100; + + ArbitraryFileListResponseInfo responseInfo = new ArbitraryFileListResponseInfo(hash58, signature58, + peer, now, arbitraryDataFileListMessage.getRequestTime(), requestHops); + + ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.add(responseInfo); + } } // Go and fetch the actual data, since this isn't a relay request diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index e60e32a9..18fd6e9c 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo; import org.qortal.data.arbitrary.ArbitraryRelayInfo; import org.qortal.data.network.ArbitraryPeerData; import org.qortal.data.network.PeerData; @@ -18,7 +19,6 @@ import org.qortal.settings.Settings; import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.Base58; import org.qortal.utils.NTP; -import org.qortal.utils.Triple; import java.security.SecureRandom; import java.util.*; @@ -45,11 +45,11 @@ public class ArbitraryDataFileManager extends Thread { public List arbitraryRelayMap = Collections.synchronizedList(new ArrayList<>()); /** - * Map to keep track of any arbitrary data file hash responses - * Key: string - the hash encoded in base58 - * Value: Triple + * List to keep track of any arbitrary data file hash responses */ - public Map> arbitraryDataFileHashResponses = Collections.synchronizedMap(new HashMap<>()); + public List arbitraryDataFileHashResponses = Collections.synchronizedList(new ArrayList<>()); + + public static int MAX_FILE_HASH_RESPONSES = 1000; private ArbitraryDataFileManager() { @@ -98,7 +98,7 @@ public class ArbitraryDataFileManager extends Thread { final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT; arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp); - arbitraryDataFileHashResponses.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < relayMinimumTimestamp); + arbitraryDataFileHashResponses.removeIf(entry -> entry.getTimestamp() < relayMinimumTimestamp); } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java index e46fd2fb..4073b6e6 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -3,6 +3,7 @@ package org.qortal.controller.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.network.Peer; import org.qortal.repository.DataException; @@ -11,11 +12,9 @@ import org.qortal.repository.RepositoryManager; import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.Base58; import org.qortal.utils.NTP; -import org.qortal.utils.Triple; -import java.util.Arrays; -import java.util.Iterator; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; public class ArbitraryDataFileRequestThread implements Runnable { @@ -51,45 +50,49 @@ public class ArbitraryDataFileRequestThread implements Runnable { boolean shouldProcess = false; synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) { - Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.entrySet().iterator(); - while (iterator.hasNext()) { - if (Controller.isStopping()) { - return; - } - - Map.Entry entry = (Map.Entry) iterator.next(); - if (entry == null || entry.getKey() == null || entry.getValue() == null) { - iterator.remove(); - continue; - } - - hash58 = (String) entry.getKey(); - Triple value = (Triple) entry.getValue(); - if (value == null) { - iterator.remove(); - continue; - } - - peer = value.getA(); - signature58 = value.getB(); - Long timestamp = value.getC(); - - if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { - // Ignore - to be deleted + if (!arbitraryDataFileManager.arbitraryDataFileHashResponses.isEmpty()) { + + // Sort by lowest number of node hops first + Comparator lowestHopsFirstComparator = + Comparator.comparingInt(ArbitraryFileListResponseInfo::getRequestHops); + arbitraryDataFileManager.arbitraryDataFileHashResponses = arbitraryDataFileManager.arbitraryDataFileHashResponses + .stream().sorted(lowestHopsFirstComparator) + .collect(Collectors.toCollection(() -> Collections.synchronizedList(new ArrayList<>()))); + + Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.iterator(); + while (iterator.hasNext()) { + if (Controller.isStopping()) { + return; + } + + ArbitraryFileListResponseInfo responseInfo = (ArbitraryFileListResponseInfo) iterator.next(); + if (responseInfo == null) { + iterator.remove(); + continue; + } + + hash58 = responseInfo.getHash58(); + peer = responseInfo.getPeer(); + signature58 = responseInfo.getSignature58(); + Long timestamp = responseInfo.getTimestamp(); + + if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { + // Ignore - to be deleted + iterator.remove(); + continue; + } + + // Skip if already requesting, but don't remove, as we might want to retry later + if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) { + // Already requesting - leave this attempt for later + continue; + } + + // We want to process this file + shouldProcess = true; iterator.remove(); - continue; + break; } - - // Skip if already requesting, but don't remove, as we might want to retry later - if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) { - // Already requesting - leave this attempt for later - continue; - } - - // We want to process this file - shouldProcess = true; - iterator.remove(); - break; } } diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryFileListResponseInfo.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryFileListResponseInfo.java new file mode 100644 index 00000000..51d1255c --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryFileListResponseInfo.java @@ -0,0 +1,11 @@ +package org.qortal.data.arbitrary; + +import org.qortal.network.Peer; + +public class ArbitraryFileListResponseInfo extends ArbitraryRelayInfo { + + public ArbitraryFileListResponseInfo(String hash58, String signature58, Peer peer, Long timestamp, Long requestTime, Integer requestHops) { + super(hash58, signature58, peer, timestamp, requestTime, requestHops); + } + +}