From f4b06fb834172312f05b9529dacf746a7372fa63 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 12 Dec 2021 10:51:24 +0000 Subject: [PATCH] Added relay mode for file list requests This reuses most of the code already in place in the core related to forwarding. - A node can opt into relay mode via the "relayModeEnabled": true setting - From this time onwards, they will ask their peers if they ever receive a file list request that they cannot serve by themselves - Whenever a peer responds with a file list, it is forwarded on to the originally requesting peer, complete with the peer address of the node that responded - The original peer can then make a request for the data file(s) themselves using a similar approach, specifying the IP address of the ultimate peer so that the relay node knows who to ask. This part is not implemented yet. --- .../arbitrary/ArbitraryDataManager.java | 75 +++++++++++-------- .../message/ArbitraryDataFileListMessage.java | 30 ++++++-- .../java/org/qortal/settings/Settings.java | 7 ++ 3 files changed, 74 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 2902f90d..cebdcd6b 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -732,15 +732,7 @@ public class ArbitraryDataManager extends Thread { public void onNetworkGetArbitraryDataMessage(Peer peer, Message message) { GetArbitraryDataMessage getArbitraryDataMessage = (GetArbitraryDataMessage) message; - byte[] signature = getArbitraryDataMessage.getSignature(); - String signature58 = Base58.encode(signature); - Long timestamp = NTP.getTime(); - Triple newEntry = new Triple<>(signature58, peer, timestamp); - - // If we've seen this request recently, then ignore - if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) - return; // Do we even have this transaction? try (final Repository repository = RepositoryManager.getRepository()) { @@ -756,10 +748,6 @@ public class ArbitraryDataManager extends Thread { if (data == null) return; - // Update requests map to reflect that we've sent it - newEntry = new Triple<>(signature58, null, timestamp); - arbitraryDataFileListRequests.put(message.getId(), newEntry); - Message arbitraryDataMessage = new ArbitraryDataMessage(signature, data); arbitraryDataMessage.setId(message.getId()); if (!peer.sendMessage(arbitraryDataMessage)) @@ -777,10 +765,12 @@ public class ArbitraryDataManager extends Thread { public void onNetworkArbitraryDataFileListMessage(Peer peer, Message message) { ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; - LOGGER.info("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size()); + String sourcePeer = arbitraryDataFileListMessage.getPeerAddress(); + LOGGER.info("Received hash list from peer {} with {} hashes, source peer: {}", peer, arbitraryDataFileListMessage.getHashes().size(), sourcePeer); // Do we have a pending request for this data? Triple request = arbitraryDataFileListRequests.get(message.getId()); + boolean isRelayRequest = (request.getB() != null); if (request == null || request.getA() == null) { return; } @@ -825,22 +815,27 @@ public class ArbitraryDataManager extends Thread { Triple newEntry = new Triple<>(null, null, request.getC()); arbitraryDataFileListRequests.put(message.getId(), newEntry); - // Go and fetch the actual data - this.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, hashes); - // FUTURE: handle response + if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) { + // Go and fetch the actual data, since this isn't a relay request + this.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, hashes); + } } catch (DataException e) { LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e); } -// // Forwarding (not yet used) -// Peer requestingPeer = request.getB(); -// if (requestingPeer != null) { -// // Forward to requesting peer; -// if (!requestingPeer.sendMessage(arbitraryDataFileListMessage)) { -// requestingPeer.disconnect("failed to forward arbitrary data file list"); -// } -// } + // Forwarding + if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) { + Peer requestingPeer = request.getB(); + if (requestingPeer != null) { + // Add the source peer's address + arbitraryDataFileListMessage.setPeerAddress(peer.getPeerData().getAddress().toString()); + // Forward to requesting peer; + if (!requestingPeer.sendMessage(arbitraryDataFileListMessage)) { + requestingPeer.disconnect("failed to forward arbitrary data file list"); + } + } + } } public void onNetworkGetArbitraryDataFileMessage(Peer peer, Message message) { @@ -886,9 +881,18 @@ public class ArbitraryDataManager extends Thread { } public void onNetworkGetArbitraryDataFileListMessage(Peer peer, Message message) { + Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet(); + GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; byte[] signature = getArbitraryDataFileListMessage.getSignature(); - Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet(); + String signature58 = Base58.encode(signature); + Long timestamp = NTP.getTime(); + Triple newEntry = new Triple<>(signature58, peer, timestamp); + + // If we've seen this request recently, then ignore + if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) { + return; + } LOGGER.info("Received hash list request from peer {} for signature {}", peer, Base58.encode(signature)); @@ -937,13 +941,22 @@ public class ArbitraryDataManager extends Thread { LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e); } - ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); - arbitraryDataFileListMessage.setId(message.getId()); - if (!peer.sendMessage(arbitraryDataFileListMessage)) { - LOGGER.info("Couldn't send list of hashes"); - peer.disconnect("failed to send list of hashes"); + // We should only respond if we have at least one hash + if (hashes.size() > 0) { + + // Update requests map to reflect that we've sent it + newEntry = new Triple<>(signature58, null, timestamp); + arbitraryDataFileListRequests.put(message.getId(), newEntry); + + ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, null, hashes); + arbitraryDataFileListMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryDataFileListMessage)) { + LOGGER.info("Couldn't send list of hashes"); + peer.disconnect("failed to send list of hashes"); + } + LOGGER.info("Sent list of hashes (count: {})", hashes.size()); + } - LOGGER.info("Sent list of hashes (count: {})", hashes.size()); } public void onNetworkArbitrarySignaturesMessage(Peer peer, Message message) { diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java index 36658a9f..0fe43265 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java @@ -1,9 +1,9 @@ package org.qortal.network.message; import com.google.common.primitives.Ints; -import org.qortal.data.block.BlockSummaryData; +import org.qortal.transform.TransformationException; import org.qortal.transform.Transformer; -import org.qortal.transform.block.BlockTransformer; +import org.qortal.utils.Serialization; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -19,18 +19,21 @@ public class ArbitraryDataFileListMessage extends Message { private final byte[] signature; private final List hashes; + private String peerAddress; - public ArbitraryDataFileListMessage(byte[] signature, List hashes) { + public ArbitraryDataFileListMessage(byte[] signature, String peerAddress, List hashes) { super(MessageType.ARBITRARY_DATA_FILE_LIST); this.signature = signature; + this.peerAddress = peerAddress; this.hashes = hashes; } - public ArbitraryDataFileListMessage(int id, byte[] signature, List hashes) { + public ArbitraryDataFileListMessage(int id, byte[] signature, String peerAddress, List hashes) { super(id, MessageType.ARBITRARY_DATA_FILE_LIST); this.signature = signature; + this.peerAddress = peerAddress; this.hashes = hashes; } @@ -42,10 +45,21 @@ public class ArbitraryDataFileListMessage extends Message { return this.signature; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public void setPeerAddress(String peerAddress) { + this.peerAddress = peerAddress; + } + + public String getPeerAddress() { + return this.peerAddress; + } + + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException, TransformationException { byte[] signature = new byte[SIGNATURE_LENGTH]; bytes.get(signature); + String peerAddress = Serialization.deserializeSizedString(bytes, 255); + int count = bytes.getInt(); if (bytes.remaining() != count * HASH_LENGTH) @@ -59,7 +73,7 @@ public class ArbitraryDataFileListMessage extends Message { hashes.add(hash); } - return new ArbitraryDataFileListMessage(id, signature, hashes); + return new ArbitraryDataFileListMessage(id, signature, peerAddress, hashes); } @Override @@ -69,6 +83,8 @@ public class ArbitraryDataFileListMessage extends Message { bytes.write(this.signature); + Serialization.serializeSizedString(bytes, this.peerAddress); + bytes.write(Ints.toByteArray(this.hashes.size())); for (byte[] hash : this.hashes) { @@ -82,7 +98,7 @@ public class ArbitraryDataFileListMessage extends Message { } public ArbitraryDataFileListMessage cloneWithNewId(int newId) { - ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes); + ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.peerAddress, this.hashes); clone.setId(newId); return clone; } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 6a137bfd..d11aff40 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -279,6 +279,9 @@ public class Settings { /** Storage policy to indicate which data should be hosted */ private String storagePolicy = "FOLLOWED_AND_VIEWED"; + /** Whether to allow data outside of the storage policy to be relayed between other peers */ + private boolean relayModeEnabled = false; + /** Expiry time (ms) for (unencrypted) built/cached data */ private Long builtDataExpiryInterval = 30 * 24 * 60 * 60 * 1000L; // 30 days @@ -828,6 +831,10 @@ public class Settings { return StoragePolicy.valueOf(this.storagePolicy); } + public boolean isRelayModeEnabled() { + return this.relayModeEnabled; + } + public Long getBuiltDataExpiryInterval() { return this.builtDataExpiryInterval; }