From e835f6d998b19bfe7300d2b06487e807928dc8f2 Mon Sep 17 00:00:00 2001 From: catbref Date: Mon, 14 Mar 2022 21:26:19 +0000 Subject: [PATCH 01/14] ExecuteProduceConsume: Slight reworking of EPC to simplify when producer can block and generally make some of the conditional code more readable. Improved logging with task class names and logging level editable during runtime! Use /peer/enginestats?newLoggingLevel=DEBUG (or TRACE or back to INFO) to change. --- .../qortal/api/resource/PeersResource.java | 27 ++++- src/main/java/org/qortal/network/Network.java | 3 +- .../qortal/utils/ExecuteProduceConsume.java | 114 ++++++++---------- 3 files changed, 79 insertions(+), 65 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/PeersResource.java b/src/main/java/org/qortal/api/resource/PeersResource.java index 1c7947c6..d89f99c4 100644 --- a/src/main/java/org/qortal/api/resource/PeersResource.java +++ b/src/main/java/org/qortal/api/resource/PeersResource.java @@ -20,6 +20,11 @@ import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.LoggerContext; import org.qortal.api.*; import org.qortal.api.model.ConnectedPeer; import org.qortal.api.model.PeersSummary; @@ -127,9 +132,29 @@ public class PeersResource { } ) @SecurityRequirement(name = "apiKey") - public ExecuteProduceConsume.StatsSnapshot getEngineStats(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + public ExecuteProduceConsume.StatsSnapshot getEngineStats(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @QueryParam("newLoggingLevel") Level newLoggingLevel) { Security.checkApiCallAllowed(request); + if (newLoggingLevel != null) { + final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + final Configuration config = ctx.getConfiguration(); + + String epcClassName = "org.qortal.network.Network.NetworkProcessor"; + LoggerConfig loggerConfig = config.getLoggerConfig(epcClassName); + LoggerConfig specificConfig = loggerConfig; + + // We need a specific configuration for this logger, + // otherwise we would change the level of all other loggers + // having the original configuration as parent as well + if (!loggerConfig.getName().equals(epcClassName)) { + specificConfig = new LoggerConfig(epcClassName, newLoggingLevel, true); + specificConfig.setParent(loggerConfig); + config.addLogger(epcClassName, specificConfig); + } + specificConfig.setLevel(newLoggingLevel); + ctx.updateLoggers(); + } + return Network.getInstance().getStatsSnapshot(); } diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index d4435ddb..20931483 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -180,7 +180,8 @@ public class Network { } // Load all known peers from repository - synchronized (this.allKnownPeers) { List fixedNetwork = Settings.getInstance().getFixedNetwork(); + synchronized (this.allKnownPeers) { + List fixedNetwork = Settings.getInstance().getFixedNetwork(); if (fixedNetwork != null && !fixedNetwork.isEmpty()) { Long addedWhen = NTP.getTime(); String addedBy = "fixedNetwork"; diff --git a/src/main/java/org/qortal/utils/ExecuteProduceConsume.java b/src/main/java/org/qortal/utils/ExecuteProduceConsume.java index 57caab9c..d8e4dbf3 100644 --- a/src/main/java/org/qortal/utils/ExecuteProduceConsume.java +++ b/src/main/java/org/qortal/utils/ExecuteProduceConsume.java @@ -28,7 +28,6 @@ public abstract class ExecuteProduceConsume implements Runnable { private final String className; private final Logger logger; - private final boolean isLoggerTraceEnabled; protected ExecutorService executor; @@ -43,12 +42,12 @@ public abstract class ExecuteProduceConsume implements Runnable { private volatile int tasksConsumed = 0; private volatile int spawnFailures = 0; + /** Whether a new thread has already been spawned and is waiting to start. Used to prevent spawning multiple new threads. */ private volatile boolean hasThreadPending = false; public ExecuteProduceConsume(ExecutorService executor) { this.className = this.getClass().getSimpleName(); this.logger = LogManager.getLogger(this.getClass()); - this.isLoggerTraceEnabled = this.logger.isTraceEnabled(); this.executor = executor; } @@ -105,8 +104,7 @@ public abstract class ExecuteProduceConsume implements Runnable { @Override public void run() { - if (this.isLoggerTraceEnabled) - Thread.currentThread().setName(this.className + "-" + Thread.currentThread().getId()); + Thread.currentThread().setName(this.className + "-" + Thread.currentThread().getId()); boolean wasThreadPending; synchronized (this) { @@ -114,25 +112,19 @@ public abstract class ExecuteProduceConsume implements Runnable { if (this.activeThreadCount > this.greatestActiveThreadCount) this.greatestActiveThreadCount = this.activeThreadCount; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] started, hasThreadPending was: %b, activeThreadCount now: %d", - Thread.currentThread().getId(), this.hasThreadPending, this.activeThreadCount)); - } + this.logger.trace(() -> String.format("[%d] started, hasThreadPending was: %b, activeThreadCount now: %d", + Thread.currentThread().getId(), this.hasThreadPending, this.activeThreadCount)); // Defer clearing hasThreadPending to prevent unnecessary threads waiting to produce... wasThreadPending = this.hasThreadPending; } try { - // It's possible this might need to become a class instance private volatile - boolean canBlock = false; - while (!Thread.currentThread().isInterrupted()) { Task task = null; + String taskType; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] waiting to produce...", Thread.currentThread().getId())); - } + this.logger.trace(() -> String.format("[%d] waiting to produce...", Thread.currentThread().getId())); synchronized (this) { if (wasThreadPending) { @@ -141,13 +133,13 @@ public abstract class ExecuteProduceConsume implements Runnable { wasThreadPending = false; } - final boolean lambdaCanIdle = canBlock; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] producing, activeThreadCount: %d, consumerCount: %d, canBlock is %b...", - Thread.currentThread().getId(), this.activeThreadCount, this.consumerCount, lambdaCanIdle)); - } + // If we're the only non-consuming thread - producer can afford to block this round + boolean canBlock = this.activeThreadCount - this.consumerCount <= 1; - final long beforeProduce = isLoggerTraceEnabled ? System.currentTimeMillis() : 0; + this.logger.trace(() -> String.format("[%d] producing... [activeThreadCount: %d, consumerCount: %d, canBlock: %b]", + Thread.currentThread().getId(), this.activeThreadCount, this.consumerCount, canBlock)); + + final long beforeProduce = this.logger.isDebugEnabled() ? System.currentTimeMillis() : 0; try { task = produceTask(canBlock); @@ -158,31 +150,36 @@ public abstract class ExecuteProduceConsume implements Runnable { this.logger.warn(() -> String.format("[%d] exception while trying to produce task", Thread.currentThread().getId()), e); } - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] producing took %dms", Thread.currentThread().getId(), System.currentTimeMillis() - beforeProduce)); + if (this.logger.isDebugEnabled()) { + final long productionPeriod = System.currentTimeMillis() - beforeProduce; + taskType = task == null ? "no task" : task.getClass().getCanonicalName(); + + this.logger.debug(() -> String.format("[%d] produced [%s] in %dms [canBlock: %b]", + Thread.currentThread().getId(), + taskType, + productionPeriod, + canBlock + )); + } else { + taskType = null; } } if (task == null) synchronized (this) { - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] no task, activeThreadCount: %d, consumerCount: %d", - Thread.currentThread().getId(), this.activeThreadCount, this.consumerCount)); - } + this.logger.trace(() -> String.format("[%d] no task, activeThreadCount: %d, consumerCount: %d", + Thread.currentThread().getId(), this.activeThreadCount, this.consumerCount)); - if (this.activeThreadCount > this.consumerCount + 1) { + // If we have an excess of non-consuming threads then we can exit + if (this.activeThreadCount - this.consumerCount > 1) { --this.activeThreadCount; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] ending, activeThreadCount now: %d", - Thread.currentThread().getId(), this.activeThreadCount)); - } + + this.logger.trace(() -> String.format("[%d] ending, activeThreadCount now: %d", + Thread.currentThread().getId(), this.activeThreadCount)); return; } - // We're the last surviving thread - producer can afford to block next round - canBlock = true; - continue; } @@ -192,16 +189,13 @@ public abstract class ExecuteProduceConsume implements Runnable { ++this.tasksProduced; ++this.consumerCount; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] hasThreadPending: %b, activeThreadCount: %d, consumerCount now: %d", - Thread.currentThread().getId(), this.hasThreadPending, this.activeThreadCount, this.consumerCount)); - } + this.logger.trace(() -> String.format("[%d] hasThreadPending: %b, activeThreadCount: %d, consumerCount now: %d", + Thread.currentThread().getId(), this.hasThreadPending, this.activeThreadCount, this.consumerCount)); // If we have no thread pending and no excess of threads then we should spawn a fresh thread - if (!this.hasThreadPending && this.activeThreadCount <= this.consumerCount + 1) { - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] spawning another thread", Thread.currentThread().getId())); - } + if (!this.hasThreadPending && this.activeThreadCount == this.consumerCount) { + this.logger.trace(() -> String.format("[%d] spawning another thread", Thread.currentThread().getId())); + this.hasThreadPending = true; try { @@ -209,21 +203,19 @@ public abstract class ExecuteProduceConsume implements Runnable { } catch (RejectedExecutionException e) { ++this.spawnFailures; this.hasThreadPending = false; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] failed to spawn another thread", Thread.currentThread().getId())); - } + + this.logger.trace(() -> String.format("[%d] failed to spawn another thread", Thread.currentThread().getId())); + this.onSpawnFailure(); } } else { - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] NOT spawning another thread", Thread.currentThread().getId())); - } + this.logger.trace(() -> String.format("[%d] NOT spawning another thread", Thread.currentThread().getId())); } } - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] performing task...", Thread.currentThread().getId())); - } + this.logger.trace(() -> String.format("[%d] consuming [%s] task...", Thread.currentThread().getId(), taskType)); + + final long beforePerform = this.logger.isDebugEnabled() ? System.currentTimeMillis() : 0; try { task.perform(); // This can block for a while @@ -231,29 +223,25 @@ public abstract class ExecuteProduceConsume implements Runnable { // We're in shutdown situation so exit Thread.currentThread().interrupt(); } catch (Exception e) { - this.logger.warn(() -> String.format("[%d] exception while performing task", Thread.currentThread().getId()), e); + this.logger.warn(() -> String.format("[%d] exception while consuming task", Thread.currentThread().getId()), e); } - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] finished task", Thread.currentThread().getId())); + if (this.logger.isDebugEnabled()) { + final long productionPeriod = System.currentTimeMillis() - beforePerform; + + this.logger.debug(() -> String.format("[%d] consumed [%s] task in %dms", Thread.currentThread().getId(), taskType, productionPeriod)); } synchronized (this) { ++this.tasksConsumed; --this.consumerCount; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] consumerCount now: %d", - Thread.currentThread().getId(), this.consumerCount)); - } - - // Quicker, non-blocking produce next round - canBlock = false; + this.logger.trace(() -> String.format("[%d] consumerCount now: %d", + Thread.currentThread().getId(), this.consumerCount)); } } } finally { - if (this.isLoggerTraceEnabled) - Thread.currentThread().setName(this.className); + Thread.currentThread().setName(this.className); } } From a5fb0be274da95fefe6e64ce765c13684621e900 Mon Sep 17 00:00:00 2001 From: catbref Date: Tue, 15 Mar 2022 20:59:56 +0000 Subject: [PATCH 02/14] Fix Network.disconnectPeer(PeerAddress) to prevent removeIf() on UnmodifiableList throwing UnsupportedOperationException --- src/main/java/org/qortal/network/Network.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 20931483..20aa10d1 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -1303,8 +1303,9 @@ public class Network { try { InetSocketAddress knownAddress = peerAddress.toSocketAddress(); - List peers = this.getImmutableConnectedPeers(); - peers.removeIf(peer -> !Peer.addressEquals(knownAddress, peer.getResolvedAddress())); + List peers = this.getImmutableConnectedPeers().stream() + .filter(peer -> Peer.addressEquals(knownAddress, peer.getResolvedAddress())) + .collect(Collectors.toList()); for (Peer peer : peers) { peer.disconnect("to be forgotten"); From 6255b2a907c42ad00067e9e0338e5f2e1808b264 Mon Sep 17 00:00:00 2001 From: catbref Date: Sat, 19 Mar 2022 13:16:32 +0000 Subject: [PATCH 03/14] Networking work-in-progress: When node has reached max connections, Network will ignore pending incoming connections by: 1. not calling accept() 2. de-registering OP_ACCEPT 'interest op' on the listen socket's channel When a peer disconnects, Network might re-register OP_ACCEPT interest op on listen socket. --- src/main/java/org/qortal/network/Network.java | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 20aa10d1..44960ccc 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -43,7 +43,7 @@ public class Network { private static final Logger LOGGER = LogManager.getLogger(Network.class); private static Network instance; - private static final int LISTEN_BACKLOG = 10; + private static final int LISTEN_BACKLOG = 5; /** * How long before retrying after a connection failure, in milliseconds. */ @@ -122,6 +122,7 @@ public class Network { private final ExecuteProduceConsume networkEPC; private Selector channelSelector; private ServerSocketChannel serverChannel; + private SelectionKey serverSelectionKey; private Iterator channelIterator = null; // volatile because value is updated inside any one of the EPC threads @@ -170,7 +171,7 @@ public class Network { serverChannel.configureBlocking(false); serverChannel.setOption(StandardSocketOptions.SO_REUSEADDR, true); serverChannel.bind(endpoint, LISTEN_BACKLOG); - serverChannel.register(channelSelector, SelectionKey.OP_ACCEPT); + serverSelectionKey = serverChannel.register(channelSelector, SelectionKey.OP_ACCEPT); } catch (UnknownHostException e) { LOGGER.error("Can't bind listen socket to address {}", Settings.getInstance().getBindAddress()); throw new IOException("Can't bind listen socket to address", e); @@ -657,6 +658,15 @@ public class Network { SocketChannel socketChannel; try { + if (getImmutableConnectedPeers().size() >= maxPeers) { + // We have enough peers + if (serverSelectionKey.interestOps() != 0) { + LOGGER.debug("Ignoring pending incoming connections because the server is full"); + serverSelectionKey.interestOps(0); + } + return; + } + socketChannel = serverSocketChannel.accept(); } catch (IOException e) { return; @@ -688,13 +698,6 @@ public class Network { return; } - if (getImmutableConnectedPeers().size() >= maxPeers) { - // We have enough peers - LOGGER.debug("Connection discarded from peer {} because the server is full", address); - socketChannel.close(); - return; - } - LOGGER.debug("Connection accepted from peer {}", address); newPeer = new Peer(socketChannel, channelSelector); @@ -783,6 +786,10 @@ public class Network { } private boolean connectPeer(Peer newPeer) throws InterruptedException { + // NOT CORRECT: + if (getImmutableConnectedPeers().size() >= minOutboundPeers) + return false; + SocketChannel socketChannel = newPeer.connect(this.channelSelector); if (socketChannel == null) { return false; @@ -866,6 +873,15 @@ public class Network { } this.removeConnectedPeer(peer); + + if (getImmutableConnectedPeers().size() < maxPeers - 1 && (serverSelectionKey.interestOps() & SelectionKey.OP_ACCEPT) == 0) { + try { + LOGGER.debug("Re-enabling accepting incoming connections because the server is not longer full"); + serverSelectionKey.interestOps(SelectionKey.OP_ACCEPT); + } catch (CancelledKeyException e) { + LOGGER.error("Failed to re-enable accepting of incoming connections: {}", e.getMessage()); + } + } } public void peerMisbehaved(Peer peer) { From b0e625907324a3117cf94f3f34744018e45bc9d5 Mon Sep 17 00:00:00 2001 From: catbref Date: Sat, 19 Mar 2022 13:19:29 +0000 Subject: [PATCH 04/14] Networking work-in-progress: De-register a peer's socket channel OP_READ interest op when producing a ChannelTask for that peer. This should prevent duplicate ChannelTasks for the same peer. Re-register OP_READ once node has read from peer's channel. --- src/main/java/org/qortal/network/Network.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 44960ccc..4b72e60e 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -596,6 +596,10 @@ public class Network { try { peer.readChannel(); + + LOGGER.trace("Thread {} re-registering OP_READ interestOps on channel: {}", + Thread.currentThread().getId(), socketChannel); + socketChannel.register(channelSelector, SelectionKey.OP_READ); } catch (IOException e) { if (e.getMessage() != null && e.getMessage().toLowerCase().contains("connection reset")) { peer.disconnect("Connection reset"); @@ -637,6 +641,12 @@ public class Network { if (channelIterator.hasNext()) { nextSelectionKey = channelIterator.next(); channelIterator.remove(); + + if (nextSelectionKey.isReadable()) { + LOGGER.trace("Thread {} clearing all interestOps on channel: {}", + Thread.currentThread().getId(), nextSelectionKey.channel()); + nextSelectionKey.interestOps(0); + } } else { nextSelectionKey = null; channelIterator = null; // Nothing to do so reset iterator to cause new select From 44fc0f367d75794efeb758282a355620a2f14842 Mon Sep 17 00:00:00 2001 From: catbref Date: Sat, 19 Mar 2022 13:21:11 +0000 Subject: [PATCH 05/14] Networking work-in-progress: Temporarily increase sleep from 1ms to 100ms when waiting for outgoing socket buffer to empty. Real fix is to rewrite using an outgoing message queue and OP_WRITE interest op. --- src/main/java/org/qortal/network/Peer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index da4a70a9..21322e85 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -633,7 +633,7 @@ public class Peer { * potentially interleaving them on-the-wire, causing checksum failures * and connection loss. */ - Thread.sleep(1L); //NOSONAR squid:S2276 + Thread.sleep(100L); //NOSONAR squid:S2276 if (System.currentTimeMillis() - sendStart > timeout) { // We've taken too long to send this message From 00996b047fdb3aaab71a9202c01d08944a7df4f4 Mon Sep 17 00:00:00 2001 From: catbref Date: Sat, 19 Mar 2022 15:08:20 +0000 Subject: [PATCH 06/14] Networking work-in-progress: As per work done by szisti in PR#45: Extracted MessageException from inside Message into its own class. Extracted MessageType from inside Message into its own class. Converted reflection-based Message.fromByteBuffer method call to non-reflection, functional interface, method-reference. This should have minor performance improvement but stronger method signature and type enforcement, as well as better IDE integration. Message.fromByteBuffer method 'contract' tightened up to: 1. throw BufferUnderflowException if there are not enough bytes to deserialize message 2. throw MessageException if bytes contain invalid data 3. should not return null Message.toData method 'contract' tightened up to: 1. return null if the message has no payload to serialize 2. throw IOException directly - no need to try-catch in each subclass Several Message-subclass fields now marked 'final' as per IDE suggestion. Several Message-subclass fromByteBuffer() method signatures have changed 'throws' list. Several bytes.remaining() != some-value changed to bytes.remaining() < some-value as per new contract. Some bytes.remaining() checks removed for fixed-length messages because we can rely on ByteBuffer throwing BufferUnderflowException. Some bytes.remaining() checks retained for variable-length messages, or messages that read a large amount of data, to prevent wasted memory allocations. Other minor tidying up --- .../org/qortal/controller/Synchronizer.java | 2 +- .../arbitrary/ArbitraryDataFileManager.java | 3 +- .../java/org/qortal/network/Handshake.java | 2 +- src/main/java/org/qortal/network/Peer.java | 4 +- .../message/ArbitraryDataFileListMessage.java | 74 +++++----- .../message/ArbitraryDataFileMessage.java | 33 ++--- .../network/message/ArbitraryDataMessage.java | 32 ++--- .../message/ArbitraryMetadataMessage.java | 33 ++--- .../message/ArbitrarySignaturesMessage.java | 39 +++-- .../qortal/network/message/BlockMessage.java | 21 ++- .../message/BlockSummariesMessage.java | 34 ++--- .../network/message/CachedBlockMessage.java | 27 ++-- .../GetArbitraryDataFileListMessage.java | 61 ++++---- .../message/GetArbitraryDataFileMessage.java | 28 ++-- .../message/GetArbitraryDataMessage.java | 24 +--- .../message/GetArbitraryMetadataMessage.java | 30 ++-- .../network/message/GetBlockMessage.java | 25 +--- .../message/GetBlockSummariesMessage.java | 29 ++-- .../message/GetOnlineAccountsMessage.java | 26 ++-- .../message/GetOnlineAccountsV2Message.java | 36 ++--- .../network/message/GetPeersMessage.java | 3 +- .../message/GetSignaturesV2Message.java | 30 ++-- .../message/GetTradePresencesMessage.java | 31 ++-- .../message/GetTransactionMessage.java | 24 +--- .../GetUnconfirmedTransactionsMessage.java | 3 +- .../network/message/GoodbyeMessage.java | 6 +- .../network/message/HeightV2Message.java | 29 ++-- .../qortal/network/message/HelloMessage.java | 17 ++- .../org/qortal/network/message/Message.java | 133 +----------------- .../network/message/MessageException.java | 19 +++ .../network/message/MessageProducer.java | 8 ++ .../qortal/network/message/MessageType.java | 96 +++++++++++++ .../message/OnlineAccountsMessage.java | 29 ++-- .../message/OnlineAccountsV2Message.java | 40 ++---- .../network/message/PeersV2Message.java | 45 +++--- .../qortal/network/message/PingMessage.java | 3 +- .../qortal/network/message/PongMessage.java | 24 ++++ .../network/message/SignaturesMessage.java | 30 ++-- .../message/TradePresencesMessage.java | 35 ++--- .../network/message/TransactionMessage.java | 21 ++- .../message/TransactionSignaturesMessage.java | 30 ++-- .../test/network/OnlineAccountsTests.java | 4 +- 42 files changed, 527 insertions(+), 696 deletions(-) create mode 100644 src/main/java/org/qortal/network/message/MessageException.java create mode 100644 src/main/java/org/qortal/network/message/MessageProducer.java create mode 100644 src/main/java/org/qortal/network/message/MessageType.java create mode 100644 src/main/java/org/qortal/network/message/PongMessage.java diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index d574ef87..27106186 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -33,7 +33,7 @@ import org.qortal.network.message.GetBlockSummariesMessage; import org.qortal.network.message.GetSignaturesV2Message; import org.qortal.network.message.Message; import org.qortal.network.message.SignaturesMessage; -import org.qortal.network.message.Message.MessageType; +import org.qortal.network.message.MessageType; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index e8b161a2..977f6215 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -7,7 +7,6 @@ import org.qortal.controller.Controller; import org.qortal.data.arbitrary.ArbitraryDirectConnectionInfo; import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo; import org.qortal.data.arbitrary.ArbitraryRelayInfo; -import org.qortal.data.network.ArbitraryPeerData; import org.qortal.data.network.PeerData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.network.Network; @@ -210,7 +209,7 @@ public class ArbitraryDataFileManager extends Thread { LOGGER.debug("Received null message from peer {}", peer); return null; } - if (message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) { + if (message.getType() != MessageType.ARBITRARY_DATA_FILE) { LOGGER.debug("Received message with invalid type: {} from peer {}", message.getType(), peer); return null; } diff --git a/src/main/java/org/qortal/network/Handshake.java b/src/main/java/org/qortal/network/Handshake.java index cdcff1d7..22354cc4 100644 --- a/src/main/java/org/qortal/network/Handshake.java +++ b/src/main/java/org/qortal/network/Handshake.java @@ -13,7 +13,7 @@ import org.qortal.crypto.MemoryPoW; import org.qortal.network.message.ChallengeMessage; import org.qortal.network.message.HelloMessage; import org.qortal.network.message.Message; -import org.qortal.network.message.Message.MessageType; +import org.qortal.network.message.MessageType; import org.qortal.settings.Settings; import org.qortal.network.message.ResponseMessage; import org.qortal.utils.DaemonThreadFactory; diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index 21322e85..b1dd0ef5 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -11,8 +11,8 @@ import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; import org.qortal.network.message.ChallengeMessage; import org.qortal.network.message.Message; -import org.qortal.network.message.Message.MessageException; -import org.qortal.network.message.Message.MessageType; +import org.qortal.network.message.MessageException; +import org.qortal.network.message.MessageType; import org.qortal.network.message.PingMessage; import org.qortal.settings.Settings; import org.qortal.utils.ExecuteProduceConsume; diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java index 32ba3fa7..952010af 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java @@ -9,17 +9,12 @@ import org.qortal.utils.Serialization; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; public class ArbitraryDataFileListMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private static final int HASH_LENGTH = Transformer.SHA256_LENGTH; - private static final int MAX_PEER_ADDRESS_LENGTH = PeerData.MAX_PEER_ADDRESS_SIZE; - private final byte[] signature; private final List hashes; private Long requestTime; @@ -60,16 +55,15 @@ public class ArbitraryDataFileListMessage extends Message { return this.signature; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException, TransformationException { - byte[] signature = new byte[SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); int count = bytes.getInt(); List hashes = new ArrayList<>(); for (int i = 0; i < count; ++i) { - - byte[] hash = new byte[HASH_LENGTH]; + byte[] hash = new byte[Transformer.SHA256_LENGTH]; bytes.get(hash); hashes.add(hash); } @@ -82,51 +76,49 @@ public class ArbitraryDataFileListMessage extends Message { // The remaining fields are optional if (bytes.hasRemaining()) { + try { + requestTime = bytes.getLong(); - requestTime = bytes.getLong(); + requestHops = bytes.getInt(); - requestHops = bytes.getInt(); - - peerAddress = Serialization.deserializeSizedStringV2(bytes, MAX_PEER_ADDRESS_LENGTH); - - isRelayPossible = bytes.getInt() > 0; + peerAddress = Serialization.deserializeSizedStringV2(bytes, PeerData.MAX_PEER_ADDRESS_SIZE); + isRelayPossible = bytes.getInt() > 0; + } catch (TransformationException e) { + throw new MessageException(e.getMessage(), e); + } } return new ArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops, peerAddress, isRelayPossible); } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(this.signature); + bytes.write(this.signature); - bytes.write(Ints.toByteArray(this.hashes.size())); + bytes.write(Ints.toByteArray(this.hashes.size())); - for (byte[] hash : this.hashes) { - bytes.write(hash); - } - - if (this.requestTime == null) { // To maintain backwards support - return bytes.toByteArray(); - } - - // The remaining fields are optional - - bytes.write(Longs.toByteArray(this.requestTime)); - - bytes.write(Ints.toByteArray(this.requestHops)); - - Serialization.serializeSizedStringV2(bytes, this.peerAddress); - - bytes.write(Ints.toByteArray(this.isRelayPossible ? 1 : 0)); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; + for (byte[] hash : this.hashes) { + bytes.write(hash); } + + if (this.requestTime == null) { // To maintain backwards support + return bytes.toByteArray(); + } + + // The remaining fields are optional + + bytes.write(Longs.toByteArray(this.requestTime)); + + bytes.write(Ints.toByteArray(this.requestHops)); + + Serialization.serializeSizedStringV2(bytes, this.peerAddress); + + bytes.write(Ints.toByteArray(this.isRelayPossible ? 1 : 0)); + + return bytes.toByteArray(); } public ArbitraryDataFileListMessage cloneWithNewId(int newId) { diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java index b9f24e29..62c5e4d4 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java @@ -9,15 +9,13 @@ import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; public class ArbitraryDataFileMessage extends Message { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileMessage.class); - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private final byte[] signature; private final ArbitraryDataFile arbitraryDataFile; @@ -39,14 +37,14 @@ public class ArbitraryDataFileMessage extends Message { return this.arbitraryDataFile; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { - byte[] signature = new byte[SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; byteBuffer.get(signature); int dataLength = byteBuffer.getInt(); - if (byteBuffer.remaining() != dataLength) - return null; + if (byteBuffer.remaining() < dataLength) + throw new BufferUnderflowException(); byte[] data = new byte[dataLength]; byteBuffer.get(data); @@ -54,15 +52,14 @@ public class ArbitraryDataFileMessage extends Message { try { ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data, signature); return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile); - } - catch (DataException e) { + } catch (DataException e) { LOGGER.info("Unable to process received file: {}", e.getMessage()); - return null; + throw new MessageException("Unable to process received file: " + e.getMessage(), e); } } @Override - protected byte[] toData() { + protected byte[] toData() throws IOException { if (this.arbitraryDataFile == null) { return null; } @@ -72,19 +69,15 @@ public class ArbitraryDataFileMessage extends Message { return null; } - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(signature); + bytes.write(signature); - bytes.write(Ints.toByteArray(data.length)); + bytes.write(Ints.toByteArray(data.length)); - bytes.write(data); + bytes.write(data); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } public ArbitraryDataFileMessage cloneWithNewId(int newId) { diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataMessage.java index 1ce149f7..b0c42942 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataMessage.java @@ -2,7 +2,7 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import org.qortal.transform.Transformer; @@ -11,10 +11,8 @@ import com.google.common.primitives.Ints; public class ArbitraryDataMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - - private byte[] signature; - private byte[] data; + private final byte[] signature; + private final byte[] data; public ArbitraryDataMessage(byte[] signature, byte[] data) { this(-1, signature, data); @@ -35,14 +33,14 @@ public class ArbitraryDataMessage extends Message { return this.data; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { - byte[] signature = new byte[SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; byteBuffer.get(signature); int dataLength = byteBuffer.getInt(); - if (byteBuffer.remaining() != dataLength) - return null; + if (byteBuffer.remaining() < dataLength) + throw new BufferUnderflowException(); byte[] data = new byte[dataLength]; byteBuffer.get(data); @@ -51,23 +49,19 @@ public class ArbitraryDataMessage extends Message { } @Override - protected byte[] toData() { + protected byte[] toData() throws IOException { if (this.data == null) return null; - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(this.signature); + bytes.write(this.signature); - bytes.write(Ints.toByteArray(this.data.length)); + bytes.write(Ints.toByteArray(this.data.length)); - bytes.write(this.data); + bytes.write(this.data); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java b/src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java index 9228d458..fb9a3790 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java @@ -7,13 +7,11 @@ import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; public class ArbitraryMetadataMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private final byte[] signature; private final ArbitraryDataFile arbitraryMetadataFile; @@ -39,14 +37,14 @@ public class ArbitraryMetadataMessage extends Message { return this.arbitraryMetadataFile; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { - byte[] signature = new byte[SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; byteBuffer.get(signature); int dataLength = byteBuffer.getInt(); - if (byteBuffer.remaining() != dataLength) - return null; + if (byteBuffer.remaining() < dataLength) + throw new BufferUnderflowException(); byte[] data = new byte[dataLength]; byteBuffer.get(data); @@ -54,14 +52,13 @@ public class ArbitraryMetadataMessage extends Message { try { ArbitraryDataFile arbitraryMetadataFile = new ArbitraryDataFile(data, signature); return new ArbitraryMetadataMessage(id, signature, arbitraryMetadataFile); - } - catch (DataException e) { - return null; + } catch (DataException e) { + throw new MessageException("Unable to process arbitrary metadata message: " + e.getMessage(), e); } } @Override - protected byte[] toData() { + protected byte[] toData() throws IOException { if (this.arbitraryMetadataFile == null) { return null; } @@ -71,19 +68,15 @@ public class ArbitraryMetadataMessage extends Message { return null; } - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(signature); + bytes.write(signature); - bytes.write(Ints.toByteArray(data.length)); + bytes.write(Ints.toByteArray(data.length)); - bytes.write(data); + bytes.write(data); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } public ArbitraryMetadataMessage cloneWithNewId(int newId) { diff --git a/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java b/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java index 1f980b3c..93c7d5e8 100644 --- a/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java @@ -8,15 +8,13 @@ import org.qortal.utils.Serialization; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; public class ArbitrarySignaturesMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private String peerAddress; private int requestHops; private List signatures; @@ -49,19 +47,24 @@ public class ArbitrarySignaturesMessage extends Message { this.requestHops = requestHops; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException, TransformationException { - String peerAddress = Serialization.deserializeSizedStringV2(bytes, PeerData.MAX_PEER_ADDRESS_SIZE); + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { + String peerAddress; + try { + peerAddress = Serialization.deserializeSizedStringV2(bytes, PeerData.MAX_PEER_ADDRESS_SIZE); + } catch (TransformationException e) { + throw new MessageException(e.getMessage(), e); + } int requestHops = bytes.getInt(); int signatureCount = bytes.getInt(); - if (bytes.remaining() != signatureCount * SIGNATURE_LENGTH) - return null; + if (bytes.remaining() < signatureCount * Transformer.SIGNATURE_LENGTH) + throw new BufferUnderflowException(); List signatures = new ArrayList<>(); for (int i = 0; i < signatureCount; ++i) { - byte[] signature = new byte[SIGNATURE_LENGTH]; + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); signatures.add(signature); } @@ -70,23 +73,19 @@ public class ArbitrarySignaturesMessage extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - Serialization.serializeSizedStringV2(bytes, this.peerAddress); + Serialization.serializeSizedStringV2(bytes, this.peerAddress); - bytes.write(Ints.toByteArray(this.requestHops)); + bytes.write(Ints.toByteArray(this.requestHops)); - bytes.write(Ints.toByteArray(this.signatures.size())); + bytes.write(Ints.toByteArray(this.signatures.size())); - for (byte[] signature : this.signatures) - bytes.write(signature); + for (byte[] signature : this.signatures) + bytes.write(signature); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/BlockMessage.java b/src/main/java/org/qortal/network/message/BlockMessage.java index b07dc8b1..8e66257f 100644 --- a/src/main/java/org/qortal/network/message/BlockMessage.java +++ b/src/main/java/org/qortal/network/message/BlockMessage.java @@ -2,7 +2,6 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.List; @@ -28,7 +27,7 @@ public class BlockMessage extends Message { private List transactions = null; private List atStates = null; - private int height; + private final int height; public BlockMessage(Block block) { super(MessageType.BLOCK); @@ -60,7 +59,7 @@ public class BlockMessage extends Message { return this.atStates; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { try { int height = byteBuffer.getInt(); @@ -72,26 +71,22 @@ public class BlockMessage extends Message { return new BlockMessage(id, blockData, blockInfo.getB(), blockInfo.getC()); } catch (TransformationException e) { LOGGER.info(String.format("Received garbled BLOCK message: %s", e.getMessage())); - return null; + throw new MessageException(e.getMessage(), e); } } @Override - protected byte[] toData() { + protected byte[] toData() throws IOException, TransformationException { if (this.block == null) return null; - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(Ints.toByteArray(this.height)); + bytes.write(Ints.toByteArray(this.height)); - bytes.write(BlockTransformer.toBytes(this.block)); + bytes.write(BlockTransformer.toBytes(this.block)); - return bytes.toByteArray(); - } catch (TransformationException | IOException e) { - return null; - } + return bytes.toByteArray(); } public BlockMessage cloneWithNewId(int newId) { diff --git a/src/main/java/org/qortal/network/message/BlockSummariesMessage.java b/src/main/java/org/qortal/network/message/BlockSummariesMessage.java index 6a30608b..56cc24e3 100644 --- a/src/main/java/org/qortal/network/message/BlockSummariesMessage.java +++ b/src/main/java/org/qortal/network/message/BlockSummariesMessage.java @@ -2,7 +2,7 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -17,7 +17,7 @@ public class BlockSummariesMessage extends Message { private static final int BLOCK_SUMMARY_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH + Transformer.INT_LENGTH + Transformer.PUBLIC_KEY_LENGTH + Transformer.INT_LENGTH; - private List blockSummaries; + private final List blockSummaries; public BlockSummariesMessage(List blockSummaries) { this(-1, blockSummaries); @@ -33,11 +33,11 @@ public class BlockSummariesMessage extends Message { return this.blockSummaries; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int count = bytes.getInt(); - if (bytes.remaining() != count * BLOCK_SUMMARY_LENGTH) - return null; + if (bytes.remaining() < count * BLOCK_SUMMARY_LENGTH) + throw new BufferUnderflowException(); List blockSummaries = new ArrayList<>(); for (int i = 0; i < count; ++i) { @@ -59,23 +59,19 @@ public class BlockSummariesMessage extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(Ints.toByteArray(this.blockSummaries.size())); + bytes.write(Ints.toByteArray(this.blockSummaries.size())); - for (BlockSummaryData blockSummary : this.blockSummaries) { - bytes.write(Ints.toByteArray(blockSummary.getHeight())); - bytes.write(blockSummary.getSignature()); - bytes.write(blockSummary.getMinterPublicKey()); - bytes.write(Ints.toByteArray(blockSummary.getOnlineAccountsCount())); - } - - return bytes.toByteArray(); - } catch (IOException e) { - return null; + for (BlockSummaryData blockSummary : this.blockSummaries) { + bytes.write(Ints.toByteArray(blockSummary.getHeight())); + bytes.write(blockSummary.getSignature()); + bytes.write(blockSummary.getMinterPublicKey()); + bytes.write(Ints.toByteArray(blockSummary.getOnlineAccountsCount())); } + + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/CachedBlockMessage.java b/src/main/java/org/qortal/network/message/CachedBlockMessage.java index e5029ab0..1a6d79d9 100644 --- a/src/main/java/org/qortal/network/message/CachedBlockMessage.java +++ b/src/main/java/org/qortal/network/message/CachedBlockMessage.java @@ -2,7 +2,6 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import org.qortal.block.Block; @@ -14,7 +13,7 @@ import com.google.common.primitives.Ints; // This is an OUTGOING-only Message which more readily lends itself to being cached public class CachedBlockMessage extends Message { - private Block block = null; + private Block block; private byte[] cachedBytes = null; public CachedBlockMessage(Block block) { @@ -30,12 +29,12 @@ public class CachedBlockMessage extends Message { this.cachedBytes = cachedBytes; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) { throw new UnsupportedOperationException("CachedBlockMessage is for outgoing messages only"); } @Override - protected byte[] toData() { + protected byte[] toData() throws IOException, TransformationException { // Already serialized? if (this.cachedBytes != null) return cachedBytes; @@ -43,22 +42,18 @@ public class CachedBlockMessage extends Message { if (this.block == null) return null; - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(Ints.toByteArray(this.block.getBlockData().getHeight())); + bytes.write(Ints.toByteArray(this.block.getBlockData().getHeight())); - bytes.write(BlockTransformer.toBytes(this.block)); + bytes.write(BlockTransformer.toBytes(this.block)); - this.cachedBytes = bytes.toByteArray(); - // We no longer need source Block - // and Block contains repository handle which is highly likely to be invalid after this call - this.block = null; + this.cachedBytes = bytes.toByteArray(); + // We no longer need source Block + // and Block contains repository handle which is highly likely to be invalid after this call + this.block = null; - return this.cachedBytes; - } catch (TransformationException | IOException e) { - return null; - } + return this.cachedBytes; } public CachedBlockMessage cloneWithNewId(int newId) { diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java index 542854a5..69668242 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java @@ -5,25 +5,16 @@ import com.google.common.primitives.Longs; import org.qortal.data.network.PeerData; import org.qortal.transform.TransformationException; import org.qortal.transform.Transformer; -import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.utils.Serialization; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -import static org.qortal.transform.Transformer.INT_LENGTH; -import static org.qortal.transform.Transformer.LONG_LENGTH; - public class GetArbitraryDataFileListMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; - private static final int MAX_PEER_ADDRESS_LENGTH = PeerData.MAX_PEER_ADDRESS_SIZE; - private final byte[] signature; private List hashes; private final long requestTime; @@ -52,8 +43,8 @@ public class GetArbitraryDataFileListMessage extends Message { return this.hashes; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException, TransformationException { - byte[] signature = new byte[SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); @@ -67,7 +58,7 @@ public class GetArbitraryDataFileListMessage extends Message { hashes = new ArrayList<>(); for (int i = 0; i < hashCount; ++i) { - byte[] hash = new byte[HASH_LENGTH]; + byte[] hash = new byte[Transformer.SHA256_LENGTH]; bytes.get(hash); hashes.add(hash); } @@ -75,42 +66,42 @@ public class GetArbitraryDataFileListMessage extends Message { String requestingPeer = null; if (bytes.hasRemaining()) { - requestingPeer = Serialization.deserializeSizedStringV2(bytes, MAX_PEER_ADDRESS_LENGTH); + try { + requestingPeer = Serialization.deserializeSizedStringV2(bytes, PeerData.MAX_PEER_ADDRESS_SIZE); + } catch (TransformationException e) { + throw new MessageException(e.getMessage(), e); + } } return new GetArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops, requestingPeer); } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(this.signature); + bytes.write(this.signature); - bytes.write(Longs.toByteArray(this.requestTime)); + bytes.write(Longs.toByteArray(this.requestTime)); - bytes.write(Ints.toByteArray(this.requestHops)); + bytes.write(Ints.toByteArray(this.requestHops)); - if (this.hashes != null) { - bytes.write(Ints.toByteArray(this.hashes.size())); + if (this.hashes != null) { + bytes.write(Ints.toByteArray(this.hashes.size())); - for (byte[] hash : this.hashes) { - bytes.write(hash); - } + for (byte[] hash : this.hashes) { + bytes.write(hash); } - else { - bytes.write(Ints.toByteArray(0)); - } - - if (this.requestingPeer != null) { - Serialization.serializeSizedStringV2(bytes, this.requestingPeer); - } - - return bytes.toByteArray(); - } catch (IOException e) { - return null; } + else { + bytes.write(Ints.toByteArray(0)); + } + + if (this.requestingPeer != null) { + Serialization.serializeSizedStringV2(bytes, this.requestingPeer); + } + + return bytes.toByteArray(); } public long getRequestTime() { diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java index 809b983d..5ec5aa82 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java @@ -1,18 +1,13 @@ package org.qortal.network.message; import org.qortal.transform.Transformer; -import org.qortal.transform.transaction.TransactionTransformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; public class GetArbitraryDataFileMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; - private final byte[] signature; private final byte[] hash; @@ -35,32 +30,25 @@ public class GetArbitraryDataFileMessage extends Message { return this.hash; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != HASH_LENGTH + SIGNATURE_LENGTH) - return null; - - byte[] signature = new byte[SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); - byte[] hash = new byte[HASH_LENGTH]; + byte[] hash = new byte[Transformer.SHA256_LENGTH]; bytes.get(hash); return new GetArbitraryDataFileMessage(id, signature, hash); } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(this.signature); + bytes.write(this.signature); - bytes.write(this.hash); + bytes.write(this.hash); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataMessage.java index 689d704b..df437477 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryDataMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataMessage.java @@ -2,16 +2,13 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import org.qortal.transform.Transformer; public class GetArbitraryDataMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - - private byte[] signature; + private final byte[] signature; public GetArbitraryDataMessage(byte[] signature) { this(-1, signature); @@ -27,11 +24,8 @@ public class GetArbitraryDataMessage extends Message { return this.signature; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != SIGNATURE_LENGTH) - return null; - - byte[] signature = new byte[SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); @@ -39,16 +33,12 @@ public class GetArbitraryDataMessage extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(this.signature); + bytes.write(this.signature); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java index 66c8f86c..a4b43e41 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java @@ -6,16 +6,10 @@ import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; -import static org.qortal.transform.Transformer.INT_LENGTH; -import static org.qortal.transform.Transformer.LONG_LENGTH; - public class GetArbitraryMetadataMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private final byte[] signature; private final long requestTime; private int requestHops; @@ -36,12 +30,8 @@ public class GetArbitraryMetadataMessage extends Message { return this.signature; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != SIGNATURE_LENGTH + LONG_LENGTH + INT_LENGTH) - return null; - - byte[] signature = new byte[SIGNATURE_LENGTH]; - + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); long requestTime = bytes.getLong(); @@ -52,20 +42,16 @@ public class GetArbitraryMetadataMessage extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(this.signature); + bytes.write(this.signature); - bytes.write(Longs.toByteArray(this.requestTime)); + bytes.write(Longs.toByteArray(this.requestTime)); - bytes.write(Ints.toByteArray(this.requestHops)); + bytes.write(Ints.toByteArray(this.requestHops)); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } public long getRequestTime() { diff --git a/src/main/java/org/qortal/network/message/GetBlockMessage.java b/src/main/java/org/qortal/network/message/GetBlockMessage.java index 43484e69..538c46cd 100644 --- a/src/main/java/org/qortal/network/message/GetBlockMessage.java +++ b/src/main/java/org/qortal/network/message/GetBlockMessage.java @@ -2,16 +2,13 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import org.qortal.transform.block.BlockTransformer; public class GetBlockMessage extends Message { - private static final int BLOCK_SIGNATURE_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH; - - private byte[] signature; + private final byte[] signature; public GetBlockMessage(byte[] signature) { this(-1, signature); @@ -27,28 +24,20 @@ public class GetBlockMessage extends Message { return this.signature; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != BLOCK_SIGNATURE_LENGTH) - return null; - - byte[] signature = new byte[BLOCK_SIGNATURE_LENGTH]; - + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] signature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; bytes.get(signature); return new GetBlockMessage(id, signature); } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(this.signature); + bytes.write(this.signature); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/GetBlockSummariesMessage.java b/src/main/java/org/qortal/network/message/GetBlockSummariesMessage.java index 148640fd..e398d532 100644 --- a/src/main/java/org/qortal/network/message/GetBlockSummariesMessage.java +++ b/src/main/java/org/qortal/network/message/GetBlockSummariesMessage.java @@ -2,20 +2,16 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; -import org.qortal.transform.Transformer; import org.qortal.transform.block.BlockTransformer; import com.google.common.primitives.Ints; public class GetBlockSummariesMessage extends Message { - private static final int BLOCK_SIGNATURE_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH; - - private byte[] parentSignature; - private int numberRequested; + private final byte[] parentSignature; + private final int numberRequested; public GetBlockSummariesMessage(byte[] parentSignature, int numberRequested) { this(-1, parentSignature, numberRequested); @@ -36,11 +32,8 @@ public class GetBlockSummariesMessage extends Message { return this.numberRequested; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != BLOCK_SIGNATURE_LENGTH + Transformer.INT_LENGTH) - return null; - - byte[] parentSignature = new byte[BLOCK_SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] parentSignature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; bytes.get(parentSignature); int numberRequested = bytes.getInt(); @@ -49,18 +42,14 @@ public class GetBlockSummariesMessage extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(this.parentSignature); + bytes.write(this.parentSignature); - bytes.write(Ints.toByteArray(this.numberRequested)); + bytes.write(Ints.toByteArray(this.numberRequested)); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java index 23c21bc5..9112ba1a 100644 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java @@ -2,7 +2,6 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -17,7 +16,7 @@ import com.google.common.primitives.Longs; public class GetOnlineAccountsMessage extends Message { private static final int MAX_ACCOUNT_COUNT = 5000; - private List onlineAccounts; + private final List onlineAccounts; public GetOnlineAccountsMessage(List onlineAccounts) { this(-1, onlineAccounts); @@ -33,7 +32,7 @@ public class GetOnlineAccountsMessage extends Message { return this.onlineAccounts; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { final int accountCount = bytes.getInt(); List onlineAccounts = new ArrayList<>(accountCount); @@ -51,23 +50,18 @@ public class GetOnlineAccountsMessage extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(Ints.toByteArray(this.onlineAccounts.size())); + bytes.write(Ints.toByteArray(this.onlineAccounts.size())); - for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); - bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); + for (OnlineAccountData onlineAccountData : this.onlineAccounts) { + bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); - bytes.write(onlineAccountData.getPublicKey()); - } - - return bytes.toByteArray(); - } catch (IOException e) { - return null; + bytes.write(onlineAccountData.getPublicKey()); } + + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java index 709f9782..bc820ff1 100644 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java @@ -7,7 +7,6 @@ import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -24,7 +23,7 @@ import java.util.Map; * Also V2 only builds online accounts message once! */ public class GetOnlineAccountsV2Message extends Message { - private List onlineAccounts; + private final List onlineAccounts; private byte[] cachedData; public GetOnlineAccountsV2Message(List onlineAccounts) { @@ -41,7 +40,7 @@ public class GetOnlineAccountsV2Message extends Message { return this.onlineAccounts; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int accountCount = bytes.getInt(); List onlineAccounts = new ArrayList<>(accountCount); @@ -68,7 +67,7 @@ public class GetOnlineAccountsV2Message extends Message { } @Override - protected synchronized byte[] toData() { + protected synchronized byte[] toData() throws IOException { if (this.cachedData != null) return this.cachedData; @@ -81,8 +80,7 @@ public class GetOnlineAccountsV2Message extends Message { // How many of each timestamp Map countByTimestamp = new HashMap<>(); - for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + for (OnlineAccountData onlineAccountData : this.onlineAccounts) { Long timestamp = onlineAccountData.getTimestamp(); countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); } @@ -91,27 +89,21 @@ public class GetOnlineAccountsV2Message extends Message { int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + this.onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH; - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - bytes.write(Longs.toByteArray(timestamp)); + bytes.write(Longs.toByteArray(timestamp)); - for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); - - if (onlineAccountData.getTimestamp() == timestamp) - bytes.write(onlineAccountData.getPublicKey()); - } + for (OnlineAccountData onlineAccountData : this.onlineAccounts) { + if (onlineAccountData.getTimestamp() == timestamp) + bytes.write(onlineAccountData.getPublicKey()); } - - this.cachedData = bytes.toByteArray(); - return this.cachedData; - } catch (IOException e) { - return null; } + + this.cachedData = bytes.toByteArray(); + return this.cachedData; } } diff --git a/src/main/java/org/qortal/network/message/GetPeersMessage.java b/src/main/java/org/qortal/network/message/GetPeersMessage.java index 21b06df5..4918b92d 100644 --- a/src/main/java/org/qortal/network/message/GetPeersMessage.java +++ b/src/main/java/org/qortal/network/message/GetPeersMessage.java @@ -1,6 +1,5 @@ package org.qortal.network.message; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; public class GetPeersMessage extends Message { @@ -13,7 +12,7 @@ public class GetPeersMessage extends Message { super(id, MessageType.GET_PEERS); } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { return new GetPeersMessage(id); } diff --git a/src/main/java/org/qortal/network/message/GetSignaturesV2Message.java b/src/main/java/org/qortal/network/message/GetSignaturesV2Message.java index 2dc54365..f0ac2265 100644 --- a/src/main/java/org/qortal/network/message/GetSignaturesV2Message.java +++ b/src/main/java/org/qortal/network/message/GetSignaturesV2Message.java @@ -2,21 +2,16 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; -import org.qortal.transform.Transformer; import org.qortal.transform.block.BlockTransformer; import com.google.common.primitives.Ints; public class GetSignaturesV2Message extends Message { - private static final int BLOCK_SIGNATURE_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH; - private static final int NUMBER_REQUESTED_LENGTH = Transformer.INT_LENGTH; - - private byte[] parentSignature; - private int numberRequested; + private final byte[] parentSignature; + private final int numberRequested; public GetSignaturesV2Message(byte[] parentSignature, int numberRequested) { this(-1, parentSignature, numberRequested); @@ -37,11 +32,8 @@ public class GetSignaturesV2Message extends Message { return this.numberRequested; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != BLOCK_SIGNATURE_LENGTH + NUMBER_REQUESTED_LENGTH) - return null; - - byte[] parentSignature = new byte[BLOCK_SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] parentSignature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; bytes.get(parentSignature); int numberRequested = bytes.getInt(); @@ -50,18 +42,14 @@ public class GetSignaturesV2Message extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(this.parentSignature); + bytes.write(this.parentSignature); - bytes.write(Ints.toByteArray(this.numberRequested)); + bytes.write(Ints.toByteArray(this.numberRequested)); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/GetTradePresencesMessage.java b/src/main/java/org/qortal/network/message/GetTradePresencesMessage.java index d9be3c1b..27fd2f9d 100644 --- a/src/main/java/org/qortal/network/message/GetTradePresencesMessage.java +++ b/src/main/java/org/qortal/network/message/GetTradePresencesMessage.java @@ -7,7 +7,6 @@ import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -20,7 +19,7 @@ import java.util.Map; * Groups of: number of entries, timestamp, then AT trade pubkey for each entry. */ public class GetTradePresencesMessage extends Message { - private List tradePresences; + private final List tradePresences; private byte[] cachedData; public GetTradePresencesMessage(List tradePresences) { @@ -37,7 +36,7 @@ public class GetTradePresencesMessage extends Message { return this.tradePresences; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int groupedEntriesCount = bytes.getInt(); List tradePresences = new ArrayList<>(groupedEntriesCount); @@ -64,7 +63,7 @@ public class GetTradePresencesMessage extends Message { } @Override - protected synchronized byte[] toData() { + protected synchronized byte[] toData() throws IOException { if (this.cachedData != null) return this.cachedData; @@ -86,25 +85,21 @@ public class GetTradePresencesMessage extends Message { int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + this.tradePresences.size() * Transformer.PUBLIC_KEY_LENGTH; - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - bytes.write(Longs.toByteArray(timestamp)); + bytes.write(Longs.toByteArray(timestamp)); - for (TradePresenceData tradePresenceData : this.tradePresences) { - if (tradePresenceData.getTimestamp() == timestamp) - bytes.write(tradePresenceData.getPublicKey()); - } + for (TradePresenceData tradePresenceData : this.tradePresences) { + if (tradePresenceData.getTimestamp() == timestamp) + bytes.write(tradePresenceData.getPublicKey()); } - - this.cachedData = bytes.toByteArray(); - return this.cachedData; - } catch (IOException e) { - return null; } + + this.cachedData = bytes.toByteArray(); + return this.cachedData; } } diff --git a/src/main/java/org/qortal/network/message/GetTransactionMessage.java b/src/main/java/org/qortal/network/message/GetTransactionMessage.java index 2ea06580..a0153aed 100644 --- a/src/main/java/org/qortal/network/message/GetTransactionMessage.java +++ b/src/main/java/org/qortal/network/message/GetTransactionMessage.java @@ -2,16 +2,13 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import org.qortal.transform.Transformer; public class GetTransactionMessage extends Message { - private static final int TRANSACTION_SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - - private byte[] signature; + private final byte[] signature; public GetTransactionMessage(byte[] signature) { this(-1, signature); @@ -27,11 +24,8 @@ public class GetTransactionMessage extends Message { return this.signature; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != TRANSACTION_SIGNATURE_LENGTH) - return null; - - byte[] signature = new byte[TRANSACTION_SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); @@ -39,16 +33,12 @@ public class GetTransactionMessage extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(this.signature); + bytes.write(this.signature); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/GetUnconfirmedTransactionsMessage.java b/src/main/java/org/qortal/network/message/GetUnconfirmedTransactionsMessage.java index 18260568..fc018e6f 100644 --- a/src/main/java/org/qortal/network/message/GetUnconfirmedTransactionsMessage.java +++ b/src/main/java/org/qortal/network/message/GetUnconfirmedTransactionsMessage.java @@ -1,6 +1,5 @@ package org.qortal.network.message; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; public class GetUnconfirmedTransactionsMessage extends Message { @@ -13,7 +12,7 @@ public class GetUnconfirmedTransactionsMessage extends Message { super(id, MessageType.GET_UNCONFIRMED_TRANSACTIONS); } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { return new GetUnconfirmedTransactionsMessage(id); } diff --git a/src/main/java/org/qortal/network/message/GoodbyeMessage.java b/src/main/java/org/qortal/network/message/GoodbyeMessage.java index 75864060..8c0c221e 100644 --- a/src/main/java/org/qortal/network/message/GoodbyeMessage.java +++ b/src/main/java/org/qortal/network/message/GoodbyeMessage.java @@ -22,7 +22,7 @@ public class GoodbyeMessage extends Message { private static final Map map = stream(Reason.values()) .collect(toMap(reason -> reason.value, reason -> reason)); - private Reason(int value) { + Reason(int value) { this.value = value; } @@ -47,12 +47,12 @@ public class GoodbyeMessage extends Message { return this.reason; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) { + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { int reasonValue = byteBuffer.getInt(); Reason reason = Reason.valueOf(reasonValue); if (reason == null) - return null; + throw new MessageException("Invalid reason " + reasonValue + " in GOODBYE message"); return new GoodbyeMessage(id, reason); } diff --git a/src/main/java/org/qortal/network/message/HeightV2Message.java b/src/main/java/org/qortal/network/message/HeightV2Message.java index 4d6f3f21..3d6a310c 100644 --- a/src/main/java/org/qortal/network/message/HeightV2Message.java +++ b/src/main/java/org/qortal/network/message/HeightV2Message.java @@ -2,7 +2,6 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import org.qortal.transform.Transformer; @@ -13,10 +12,10 @@ import com.google.common.primitives.Longs; public class HeightV2Message extends Message { - private int height; - private byte[] signature; - private long timestamp; - private byte[] minterPublicKey; + private final int height; + private final byte[] signature; + private final long timestamp; + private final byte[] minterPublicKey; public HeightV2Message(int height, byte[] signature, long timestamp, byte[] minterPublicKey) { this(-1, height, signature, timestamp, minterPublicKey); @@ -47,7 +46,7 @@ public class HeightV2Message extends Message { return this.minterPublicKey; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int height = bytes.getInt(); byte[] signature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; @@ -62,22 +61,18 @@ public class HeightV2Message extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(Ints.toByteArray(this.height)); + bytes.write(Ints.toByteArray(this.height)); - bytes.write(this.signature); + bytes.write(this.signature); - bytes.write(Longs.toByteArray(this.timestamp)); + bytes.write(Longs.toByteArray(this.timestamp)); - bytes.write(this.minterPublicKey); + bytes.write(this.minterPublicKey); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/HelloMessage.java b/src/main/java/org/qortal/network/message/HelloMessage.java index 1b6de17d..80314c2e 100644 --- a/src/main/java/org/qortal/network/message/HelloMessage.java +++ b/src/main/java/org/qortal/network/message/HelloMessage.java @@ -39,15 +39,20 @@ public class HelloMessage extends Message { return this.senderPeerAddress; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws TransformationException { + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { long timestamp = byteBuffer.getLong(); - String versionString = Serialization.deserializeSizedString(byteBuffer, 255); - - // Sender peer address added in v3.0, so is an optional field. Older versions won't send it. + String versionString; String senderPeerAddress = null; - if (byteBuffer.hasRemaining()) { - senderPeerAddress = Serialization.deserializeSizedString(byteBuffer, 255); + try { + versionString = Serialization.deserializeSizedString(byteBuffer, 255); + + // Sender peer address added in v3.0, so is an optional field. Older versions won't send it. + if (byteBuffer.hasRemaining()) { + senderPeerAddress = Serialization.deserializeSizedString(byteBuffer, 255); + } + } catch (TransformationException e) { + throw new MessageException(e.getMessage(), e); } return new HelloMessage(id, timestamp, versionString, senderPeerAddress); diff --git a/src/main/java/org/qortal/network/message/Message.java b/src/main/java/org/qortal/network/message/Message.java index b06a5133..15e8cb4f 100644 --- a/src/main/java/org/qortal/network/message/Message.java +++ b/src/main/java/org/qortal/network/message/Message.java @@ -1,7 +1,5 @@ package org.qortal.network.message; -import java.util.Map; - import org.qortal.crypto.Crypto; import org.qortal.network.Network; import org.qortal.transform.TransformationException; @@ -13,8 +11,6 @@ import static java.util.stream.Collectors.toMap; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.Arrays; @@ -27,127 +23,6 @@ public abstract class Message { private static final int MAX_DATA_SIZE = 10 * 1024 * 1024; // 10MB - @SuppressWarnings("serial") - public static class MessageException extends Exception { - public MessageException() { - } - - public MessageException(String message) { - super(message); - } - - public MessageException(String message, Throwable cause) { - super(message, cause); - } - - public MessageException(Throwable cause) { - super(cause); - } - } - - public enum MessageType { - // Handshaking - HELLO(0), - GOODBYE(1), - CHALLENGE(2), - RESPONSE(3), - - // Status / notifications - HEIGHT_V2(10), - PING(11), - PONG(12), - - // Requesting data - PEERS_V2(20), - GET_PEERS(21), - - TRANSACTION(30), - GET_TRANSACTION(31), - - TRANSACTION_SIGNATURES(40), - GET_UNCONFIRMED_TRANSACTIONS(41), - - BLOCK(50), - GET_BLOCK(51), - - SIGNATURES(60), - GET_SIGNATURES_V2(61), - - BLOCK_SUMMARIES(70), - GET_BLOCK_SUMMARIES(71), - - ONLINE_ACCOUNTS(80), - GET_ONLINE_ACCOUNTS(81), - ONLINE_ACCOUNTS_V2(82), - GET_ONLINE_ACCOUNTS_V2(83), - - ARBITRARY_DATA(90), - GET_ARBITRARY_DATA(91), - - BLOCKS(100), - GET_BLOCKS(101), - - ARBITRARY_DATA_FILE(110), - GET_ARBITRARY_DATA_FILE(111), - - ARBITRARY_DATA_FILE_LIST(120), - GET_ARBITRARY_DATA_FILE_LIST(121), - - ARBITRARY_SIGNATURES(130), - - TRADE_PRESENCES(140), - GET_TRADE_PRESENCES(141), - - ARBITRARY_METADATA(150), - GET_ARBITRARY_METADATA(151); - - public final int value; - public final Method fromByteBufferMethod; - - private static final Map map = stream(MessageType.values()) - .collect(toMap(messageType -> messageType.value, messageType -> messageType)); - - private MessageType(int value) { - this.value = value; - - String[] classNameParts = this.name().toLowerCase().split("_"); - - for (int i = 0; i < classNameParts.length; ++i) - classNameParts[i] = classNameParts[i].substring(0, 1).toUpperCase().concat(classNameParts[i].substring(1)); - - String className = String.join("", classNameParts); - - Method method; - try { - Class subclass = Class.forName(String.join("", Message.class.getPackage().getName(), ".", className, "Message")); - - method = subclass.getDeclaredMethod("fromByteBuffer", int.class, ByteBuffer.class); - } catch (ClassNotFoundException | NoSuchMethodException | SecurityException e) { - method = null; - } - - this.fromByteBufferMethod = method; - } - - public static MessageType valueOf(int value) { - return map.get(value); - } - - public Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { - if (this.fromByteBufferMethod == null) - throw new MessageException("Unsupported message type [" + value + "] during conversion from bytes"); - - try { - return (Message) this.fromByteBufferMethod.invoke(null, id, byteBuffer); - } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - if (e.getCause() instanceof BufferUnderflowException) - throw new MessageException("Byte data too short for " + name() + " message"); - - throw new MessageException("Internal error with " + name() + " message during conversion from bytes"); - } - } - } - private int id; private MessageType type; @@ -181,7 +56,7 @@ public abstract class Message { * * @param readOnlyBuffer * @return null if no complete message can be read - * @throws MessageException + * @throws MessageException if message could not be decoded or is invalid */ public static Message fromByteBuffer(ByteBuffer readOnlyBuffer) throws MessageException { try { @@ -293,6 +168,12 @@ public abstract class Message { } } + /** Serialize message into bytes. + * + * @return message as byte array, or null if message is missing payload data / uninitialized somehow + * @throws IOException if unable / failed to serialize + * @throws TransformationException if unable / failed to serialize + */ protected abstract byte[] toData() throws IOException, TransformationException; } diff --git a/src/main/java/org/qortal/network/message/MessageException.java b/src/main/java/org/qortal/network/message/MessageException.java new file mode 100644 index 00000000..97e8d0be --- /dev/null +++ b/src/main/java/org/qortal/network/message/MessageException.java @@ -0,0 +1,19 @@ +package org.qortal.network.message; + +@SuppressWarnings("serial") +public class MessageException extends Exception { + public MessageException() { + } + + public MessageException(String message) { + super(message); + } + + public MessageException(String message, Throwable cause) { + super(message, cause); + } + + public MessageException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/org/qortal/network/message/MessageProducer.java b/src/main/java/org/qortal/network/message/MessageProducer.java new file mode 100644 index 00000000..7f203788 --- /dev/null +++ b/src/main/java/org/qortal/network/message/MessageProducer.java @@ -0,0 +1,8 @@ +package org.qortal.network.message; + +import java.nio.ByteBuffer; + +@FunctionalInterface +public interface MessageProducer { + Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException; +} diff --git a/src/main/java/org/qortal/network/message/MessageType.java b/src/main/java/org/qortal/network/message/MessageType.java new file mode 100644 index 00000000..48039a4d --- /dev/null +++ b/src/main/java/org/qortal/network/message/MessageType.java @@ -0,0 +1,96 @@ +package org.qortal.network.message; + +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.util.Map; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +public enum MessageType { + // Handshaking + HELLO(0, HelloMessage::fromByteBuffer), + GOODBYE(1, GoodbyeMessage::fromByteBuffer), + CHALLENGE(2, ChallengeMessage::fromByteBuffer), + RESPONSE(3, ResponseMessage::fromByteBuffer), + + // Status / notifications + HEIGHT_V2(10, HeightV2Message::fromByteBuffer), + PING(11, PingMessage::fromByteBuffer), + PONG(12, PongMessage::fromByteBuffer), + + // Requesting data + PEERS_V2(20, PeersV2Message::fromByteBuffer), + GET_PEERS(21, GetPeersMessage::fromByteBuffer), + + TRANSACTION(30, TransactionMessage::fromByteBuffer), + GET_TRANSACTION(31, GetTransactionMessage::fromByteBuffer), + + TRANSACTION_SIGNATURES(40, TransactionSignaturesMessage::fromByteBuffer), + GET_UNCONFIRMED_TRANSACTIONS(41, GetUnconfirmedTransactionsMessage::fromByteBuffer), + + BLOCK(50, BlockMessage::fromByteBuffer), + GET_BLOCK(51, GetBlockMessage::fromByteBuffer), + + SIGNATURES(60, SignaturesMessage::fromByteBuffer), + GET_SIGNATURES_V2(61, GetSignaturesV2Message::fromByteBuffer), + + BLOCK_SUMMARIES(70, BlockSummariesMessage::fromByteBuffer), + GET_BLOCK_SUMMARIES(71, GetBlockSummariesMessage::fromByteBuffer), + + ONLINE_ACCOUNTS(80, OnlineAccountsMessage::fromByteBuffer), + GET_ONLINE_ACCOUNTS(81, GetOnlineAccountsMessage::fromByteBuffer), + ONLINE_ACCOUNTS_V2(82, OnlineAccountsV2Message::fromByteBuffer), + GET_ONLINE_ACCOUNTS_V2(83, GetOnlineAccountsV2Message::fromByteBuffer), + + ARBITRARY_DATA(90, ArbitraryDataMessage::fromByteBuffer), + GET_ARBITRARY_DATA(91, GetArbitraryDataMessage::fromByteBuffer), + + BLOCKS(100, null), // unsupported + GET_BLOCKS(101, null), // unsupported + + ARBITRARY_DATA_FILE(110, ArbitraryDataFileMessage::fromByteBuffer), + GET_ARBITRARY_DATA_FILE(111, GetArbitraryDataFileMessage::fromByteBuffer), + + ARBITRARY_DATA_FILE_LIST(120, ArbitraryDataFileListMessage::fromByteBuffer), + GET_ARBITRARY_DATA_FILE_LIST(121, GetArbitraryDataFileListMessage::fromByteBuffer), + + ARBITRARY_SIGNATURES(130, ArbitrarySignaturesMessage::fromByteBuffer), + + TRADE_PRESENCES(140, TradePresencesMessage::fromByteBuffer), + GET_TRADE_PRESENCES(141, GetTradePresencesMessage::fromByteBuffer), + + ARBITRARY_METADATA(150, ArbitraryMetadataMessage::fromByteBuffer), + GET_ARBITRARY_METADATA(151, GetArbitraryMetadataMessage::fromByteBuffer); + + public final int value; + public final MessageProducer fromByteBufferMethod; + + private static final Map map = stream(MessageType.values()) + .collect(toMap(messageType -> messageType.value, messageType -> messageType)); + + MessageType(int value, MessageProducer fromByteBufferMethod) { + this.value = value; + this.fromByteBufferMethod = fromByteBufferMethod; + } + + public static MessageType valueOf(int value) { + return map.get(value); + } + + /** + * Attempt to read a message from byte buffer. + * + * @param id message ID or -1 + * @param byteBuffer ByteBuffer source for message + * @return null if no complete message can be read + * @throws MessageException if message could not be decoded or is invalid + * @throws BufferUnderflowException if not enough bytes in buffer to read message + */ + public Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { + if (this.fromByteBufferMethod == null) + throw new MessageException("Message type " + this.name() + " unsupported"); + + return this.fromByteBufferMethod.fromByteBuffer(id, byteBuffer); + } +} diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java index 02c46717..d7acb2fd 100644 --- a/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java +++ b/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java @@ -2,7 +2,6 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -17,7 +16,7 @@ import com.google.common.primitives.Longs; public class OnlineAccountsMessage extends Message { private static final int MAX_ACCOUNT_COUNT = 5000; - private List onlineAccounts; + private final List onlineAccounts; public OnlineAccountsMessage(List onlineAccounts) { this(-1, onlineAccounts); @@ -33,7 +32,7 @@ public class OnlineAccountsMessage extends Message { return this.onlineAccounts; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { final int accountCount = bytes.getInt(); List onlineAccounts = new ArrayList<>(accountCount); @@ -55,26 +54,20 @@ public class OnlineAccountsMessage extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(Ints.toByteArray(this.onlineAccounts.size())); + bytes.write(Ints.toByteArray(this.onlineAccounts.size())); - for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + for (OnlineAccountData onlineAccountData : this.onlineAccounts) { + bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); - bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); + bytes.write(onlineAccountData.getSignature()); - bytes.write(onlineAccountData.getSignature()); - - bytes.write(onlineAccountData.getPublicKey()); - } - - return bytes.toByteArray(); - } catch (IOException e) { - return null; + bytes.write(onlineAccountData.getPublicKey()); } + + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java index f0fce81e..955237fe 100644 --- a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java +++ b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java @@ -7,13 +7,11 @@ import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * For sending online accounts info to remote peer. @@ -25,7 +23,7 @@ import java.util.stream.Collectors; * Also V2 only builds online accounts message once! */ public class OnlineAccountsV2Message extends Message { - private List onlineAccounts; + private final List onlineAccounts; private byte[] cachedData; public OnlineAccountsV2Message(List onlineAccounts) { @@ -42,7 +40,7 @@ public class OnlineAccountsV2Message extends Message { return this.onlineAccounts; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { int accountCount = bytes.getInt(); List onlineAccounts = new ArrayList<>(accountCount); @@ -72,7 +70,7 @@ public class OnlineAccountsV2Message extends Message { } @Override - protected synchronized byte[] toData() { + protected synchronized byte[] toData() throws IOException { if (this.cachedData != null) return this.cachedData; @@ -85,8 +83,7 @@ public class OnlineAccountsV2Message extends Message { // How many of each timestamp Map countByTimestamp = new HashMap<>(); - for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + for (OnlineAccountData onlineAccountData : this.onlineAccounts) { Long timestamp = onlineAccountData.getTimestamp(); countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); } @@ -95,30 +92,23 @@ public class OnlineAccountsV2Message extends Message { int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + this.onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH); - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - bytes.write(Longs.toByteArray(timestamp)); + bytes.write(Longs.toByteArray(timestamp)); - for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); - - if (onlineAccountData.getTimestamp() == timestamp) { - bytes.write(onlineAccountData.getSignature()); - - bytes.write(onlineAccountData.getPublicKey()); - } + for (OnlineAccountData onlineAccountData : this.onlineAccounts) { + if (onlineAccountData.getTimestamp() == timestamp) { + bytes.write(onlineAccountData.getSignature()); + bytes.write(onlineAccountData.getPublicKey()); } } - - this.cachedData = bytes.toByteArray(); - return this.cachedData; - } catch (IOException e) { - return null; } + + this.cachedData = bytes.toByteArray(); + return this.cachedData; } } diff --git a/src/main/java/org/qortal/network/message/PeersV2Message.java b/src/main/java/org/qortal/network/message/PeersV2Message.java index bfea87c7..4166c37d 100644 --- a/src/main/java/org/qortal/network/message/PeersV2Message.java +++ b/src/main/java/org/qortal/network/message/PeersV2Message.java @@ -2,7 +2,6 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -16,7 +15,7 @@ import com.google.common.primitives.Ints; // NOTE: this message supports hostnames, literal IP addresses (IPv4 and IPv6) with port numbers public class PeersV2Message extends Message { - private List peerAddresses; + private final List peerAddresses; public PeersV2Message(List peerAddresses) { this(-1, peerAddresses); @@ -32,7 +31,7 @@ public class PeersV2Message extends Message { return this.peerAddresses; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { // Read entry count int count = byteBuffer.getInt(); @@ -49,7 +48,7 @@ public class PeersV2Message extends Message { PeerAddress peerAddress = PeerAddress.fromString(addressString); peerAddresses.add(peerAddress); } catch (IllegalArgumentException e) { - // Not valid - ignore + throw new MessageException("Invalid peer address in received PEERS_V2 message"); } } @@ -57,35 +56,31 @@ public class PeersV2Message extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - List addresses = new ArrayList<>(); + List addresses = new ArrayList<>(); - // First entry represents sending node but contains only port number with empty address. - addresses.add(("0.0.0.0:" + Settings.getInstance().getListenPort()).getBytes(StandardCharsets.UTF_8)); + // First entry represents sending node but contains only port number with empty address. + addresses.add(("0.0.0.0:" + Settings.getInstance().getListenPort()).getBytes(StandardCharsets.UTF_8)); - for (PeerAddress peerAddress : this.peerAddresses) - addresses.add(peerAddress.toString().getBytes(StandardCharsets.UTF_8)); + for (PeerAddress peerAddress : this.peerAddresses) + addresses.add(peerAddress.toString().getBytes(StandardCharsets.UTF_8)); - // We can't send addresses that are longer than 255 bytes as length itself is encoded in one byte. - addresses.removeIf(addressString -> addressString.length > 255); + // We can't send addresses that are longer than 255 bytes as length itself is encoded in one byte. + addresses.removeIf(addressString -> addressString.length > 255); - // Serialize + // Serialize - // Number of entries - bytes.write(Ints.toByteArray(addresses.size())); + // Number of entries + bytes.write(Ints.toByteArray(addresses.size())); - for (byte[] address : addresses) { - bytes.write(address.length); - bytes.write(address); - } - - return bytes.toByteArray(); - } catch (IOException e) { - return null; + for (byte[] address : addresses) { + bytes.write(address.length); + bytes.write(address); } + + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/PingMessage.java b/src/main/java/org/qortal/network/message/PingMessage.java index ddec0fd7..fa04fdb6 100644 --- a/src/main/java/org/qortal/network/message/PingMessage.java +++ b/src/main/java/org/qortal/network/message/PingMessage.java @@ -1,6 +1,5 @@ package org.qortal.network.message; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; public class PingMessage extends Message { @@ -13,7 +12,7 @@ public class PingMessage extends Message { super(id, MessageType.PING); } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { return new PingMessage(id); } diff --git a/src/main/java/org/qortal/network/message/PongMessage.java b/src/main/java/org/qortal/network/message/PongMessage.java new file mode 100644 index 00000000..4338aeac --- /dev/null +++ b/src/main/java/org/qortal/network/message/PongMessage.java @@ -0,0 +1,24 @@ +package org.qortal.network.message; + +import java.nio.ByteBuffer; + +public class PongMessage extends Message { + + public PongMessage() { + this(-1); + } + + private PongMessage(int id) { + super(id, MessageType.PONG); + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + return new PongMessage(id); + } + + @Override + protected byte[] toData() { + return new byte[0]; + } + +} diff --git a/src/main/java/org/qortal/network/message/SignaturesMessage.java b/src/main/java/org/qortal/network/message/SignaturesMessage.java index 008f4c1a..cbbaf790 100644 --- a/src/main/java/org/qortal/network/message/SignaturesMessage.java +++ b/src/main/java/org/qortal/network/message/SignaturesMessage.java @@ -2,7 +2,7 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -13,9 +13,7 @@ import com.google.common.primitives.Ints; public class SignaturesMessage extends Message { - private static final int BLOCK_SIGNATURE_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH; - - private List signatures; + private final List signatures; public SignaturesMessage(List signatures) { this(-1, signatures); @@ -31,15 +29,15 @@ public class SignaturesMessage extends Message { return this.signatures; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int count = bytes.getInt(); - if (bytes.remaining() != count * BLOCK_SIGNATURE_LENGTH) - return null; + if (bytes.remaining() < count * BlockTransformer.BLOCK_SIGNATURE_LENGTH) + throw new BufferUnderflowException(); List signatures = new ArrayList<>(); for (int i = 0; i < count; ++i) { - byte[] signature = new byte[BLOCK_SIGNATURE_LENGTH]; + byte[] signature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; bytes.get(signature); signatures.add(signature); } @@ -48,19 +46,15 @@ public class SignaturesMessage extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(Ints.toByteArray(this.signatures.size())); + bytes.write(Ints.toByteArray(this.signatures.size())); - for (byte[] signature : this.signatures) - bytes.write(signature); + for (byte[] signature : this.signatures) + bytes.write(signature); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } } diff --git a/src/main/java/org/qortal/network/message/TradePresencesMessage.java b/src/main/java/org/qortal/network/message/TradePresencesMessage.java index 9d846722..20edfdaf 100644 --- a/src/main/java/org/qortal/network/message/TradePresencesMessage.java +++ b/src/main/java/org/qortal/network/message/TradePresencesMessage.java @@ -8,7 +8,6 @@ import org.qortal.utils.Base58; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -21,7 +20,7 @@ import java.util.Map; * Groups of: number of entries, timestamp, then pubkey + sig + AT address for each entry. */ public class TradePresencesMessage extends Message { - private List tradePresences; + private final List tradePresences; private byte[] cachedData; public TradePresencesMessage(List tradePresences) { @@ -38,7 +37,7 @@ public class TradePresencesMessage extends Message { return this.tradePresences; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int groupedEntriesCount = bytes.getInt(); List tradePresences = new ArrayList<>(groupedEntriesCount); @@ -72,7 +71,7 @@ public class TradePresencesMessage extends Message { } @Override - protected synchronized byte[] toData() { + protected synchronized byte[] toData() throws IOException { if (this.cachedData != null) return this.cachedData; @@ -94,30 +93,26 @@ public class TradePresencesMessage extends Message { int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + this.tradePresences.size() * (Transformer.PUBLIC_KEY_LENGTH + Transformer.SIGNATURE_LENGTH + Transformer.ADDRESS_LENGTH); - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - bytes.write(Longs.toByteArray(timestamp)); + bytes.write(Longs.toByteArray(timestamp)); - for (TradePresenceData tradePresenceData : this.tradePresences) { - if (tradePresenceData.getTimestamp() == timestamp) { - bytes.write(tradePresenceData.getPublicKey()); + for (TradePresenceData tradePresenceData : this.tradePresences) { + if (tradePresenceData.getTimestamp() == timestamp) { + bytes.write(tradePresenceData.getPublicKey()); - bytes.write(tradePresenceData.getSignature()); + bytes.write(tradePresenceData.getSignature()); - bytes.write(Base58.decode(tradePresenceData.getAtAddress())); - } + bytes.write(Base58.decode(tradePresenceData.getAtAddress())); } } - - this.cachedData = bytes.toByteArray(); - return this.cachedData; - } catch (IOException e) { - return null; } + + this.cachedData = bytes.toByteArray(); + return this.cachedData; } } diff --git a/src/main/java/org/qortal/network/message/TransactionMessage.java b/src/main/java/org/qortal/network/message/TransactionMessage.java index 92cce086..6aa3c96f 100644 --- a/src/main/java/org/qortal/network/message/TransactionMessage.java +++ b/src/main/java/org/qortal/network/message/TransactionMessage.java @@ -1,6 +1,5 @@ package org.qortal.network.message; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import org.qortal.data.transaction.TransactionData; @@ -25,26 +24,24 @@ public class TransactionMessage extends Message { return this.transactionData; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { - try { - TransactionData transactionData = TransactionTransformer.fromByteBuffer(byteBuffer); + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { + TransactionData transactionData; - return new TransactionMessage(id, transactionData); + try { + transactionData = TransactionTransformer.fromByteBuffer(byteBuffer); } catch (TransformationException e) { - return null; + throw new MessageException(e.getMessage(), e); } + + return new TransactionMessage(id, transactionData); } @Override - protected byte[] toData() { + protected byte[] toData() throws TransformationException { if (this.transactionData == null) return null; - try { - return TransactionTransformer.toBytes(this.transactionData); - } catch (TransformationException e) { - return null; - } + return TransactionTransformer.toBytes(this.transactionData); } } diff --git a/src/main/java/org/qortal/network/message/TransactionSignaturesMessage.java b/src/main/java/org/qortal/network/message/TransactionSignaturesMessage.java index 082a7187..9d049e20 100644 --- a/src/main/java/org/qortal/network/message/TransactionSignaturesMessage.java +++ b/src/main/java/org/qortal/network/message/TransactionSignaturesMessage.java @@ -2,7 +2,7 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -13,9 +13,7 @@ import com.google.common.primitives.Ints; public class TransactionSignaturesMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - - private List signatures; + private final List signatures; public TransactionSignaturesMessage(List signatures) { this(-1, signatures); @@ -31,15 +29,15 @@ public class TransactionSignaturesMessage extends Message { return this.signatures; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int count = bytes.getInt(); - if (bytes.remaining() != count * SIGNATURE_LENGTH) - return null; + if (bytes.remaining() < count * Transformer.SIGNATURE_LENGTH) + throw new BufferUnderflowException(); List signatures = new ArrayList<>(); for (int i = 0; i < count; ++i) { - byte[] signature = new byte[SIGNATURE_LENGTH]; + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); signatures.add(signature); } @@ -48,19 +46,15 @@ public class TransactionSignaturesMessage extends Message { } @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + protected byte[] toData() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(Ints.toByteArray(this.signatures.size())); + bytes.write(Ints.toByteArray(this.signatures.size())); - for (byte[] signature : this.signatures) - bytes.write(signature); + for (byte[] signature : this.signatures) + bytes.write(signature); - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } + return bytes.toByteArray(); } } diff --git a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java index b1c5ec4f..4154121c 100644 --- a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java +++ b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java @@ -29,7 +29,7 @@ public class OnlineAccountsTests { @Test - public void testGetOnlineAccountsV2() throws Message.MessageException { + public void testGetOnlineAccountsV2() throws MessageException { List onlineAccountsOut = generateOnlineAccounts(false); Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut); @@ -58,7 +58,7 @@ public class OnlineAccountsTests { } @Test - public void testOnlineAccountsV2() throws Message.MessageException { + public void testOnlineAccountsV2() throws MessageException { List onlineAccountsOut = generateOnlineAccounts(true); Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut); From 91e0c9b940a3c35f766ee695be2183c3a7edd84b Mon Sep 17 00:00:00 2001 From: catbref Date: Mon, 28 Mar 2022 21:11:16 +0100 Subject: [PATCH 07/14] More improvements to networking: As per work done by szisti in PR#45: Extracted network 'Tasks' to their own classes. Network.NetworkProcessor reduced to only producing Tasks. Improved usage of SocketChannel interest-ops. Eventually this might lead to reducing task-producing synchronization lock into more granular locks. Work still needed to convert sending messages to a queue and to make use of OP_WRITE instead of sleeping to wait for socket buffer to empty. Disabled PeerConnectTask producer from checking against connected peers via DNS as it's too slow. Swapped Peer's replyQueues from SynchronizedMap(wrapped HashMap) to ConcurrentHashMap. Other minor changes within networking. --- src/main/java/org/qortal/network/Network.java | 300 +++++++----------- src/main/java/org/qortal/network/Peer.java | 61 ++-- .../qortal/network/task/BroadcastTask.java | 22 ++ .../network/task/ChannelAcceptTask.java | 99 ++++++ .../qortal/network/task/ChannelReadTask.java | 55 ++++ .../qortal/network/task/ChannelWriteTask.java | 56 ++++ .../org/qortal/network/task/MessageTask.java | 28 ++ .../qortal/network/task/PeerConnectTask.java | 33 ++ .../org/qortal/network/task/PingTask.java | 44 +++ .../qortal/utils/ExecuteProduceConsume.java | 6 +- src/test/java/org/qortal/test/EPCTests.java | 153 +++++---- 11 files changed, 582 insertions(+), 275 deletions(-) create mode 100644 src/main/java/org/qortal/network/task/BroadcastTask.java create mode 100644 src/main/java/org/qortal/network/task/ChannelAcceptTask.java create mode 100644 src/main/java/org/qortal/network/task/ChannelReadTask.java create mode 100644 src/main/java/org/qortal/network/task/ChannelWriteTask.java create mode 100644 src/main/java/org/qortal/network/task/MessageTask.java create mode 100644 src/main/java/org/qortal/network/task/PeerConnectTask.java create mode 100644 src/main/java/org/qortal/network/task/PingTask.java diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 4b72e60e..3ee7af75 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -13,6 +13,7 @@ import org.qortal.data.block.BlockData; import org.qortal.data.network.PeerData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.message.*; +import org.qortal.network.task.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -216,12 +217,16 @@ public class Network { // Getters / setters - public static synchronized Network getInstance() { - if (instance == null) { - instance = new Network(); - } + private static class SingletonContainer { + private static final Network INSTANCE = new Network(); + } - return instance; + public static Network getInstance() { + return SingletonContainer.INSTANCE; + } + + public int getMaxPeers() { + return this.maxPeers; } public byte[] getMessageMagic() { @@ -496,39 +501,19 @@ public class Network { } private Task maybeProducePeerMessageTask() { - for (Peer peer : getImmutableConnectedPeers()) { - Task peerTask = peer.getMessageTask(); - if (peerTask != null) { - return peerTask; - } - } - - return null; + return getImmutableConnectedPeers().stream() + .map(Peer::getMessageTask) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); } private Task maybeProducePeerPingTask(Long now) { - // Ask connected peers whether they need a ping - for (Peer peer : getImmutableHandshakedPeers()) { - Task peerTask = peer.getPingTask(now); - if (peerTask != null) { - return peerTask; - } - } - - return null; - } - - class PeerConnectTask implements ExecuteProduceConsume.Task { - private final Peer peer; - - PeerConnectTask(Peer peer) { - this.peer = peer; - } - - @Override - public void perform() throws InterruptedException { - connectPeer(peer); - } + return getImmutableHandshakedPeers().stream() + .map(peer -> peer.getPingTask(now)) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); } private Task maybeProduceConnectPeerTask(Long now) throws InterruptedException { @@ -557,65 +542,10 @@ public class Network { } nextBroadcastTimestamp = now + BROADCAST_INTERVAL; - return () -> Controller.getInstance().doNetworkBroadcast(); - } - - class ChannelTask implements ExecuteProduceConsume.Task { - private final SelectionKey selectionKey; - - ChannelTask(SelectionKey selectionKey) { - this.selectionKey = selectionKey; - } - - @Override - public void perform() throws InterruptedException { - try { - LOGGER.trace("Thread {} has pending channel: {}, with ops {}", - Thread.currentThread().getId(), selectionKey.channel(), selectionKey.readyOps()); - - // process pending channel task - if (selectionKey.isReadable()) { - connectionRead((SocketChannel) selectionKey.channel()); - } else if (selectionKey.isAcceptable()) { - acceptConnection((ServerSocketChannel) selectionKey.channel()); - } - - LOGGER.trace("Thread {} processed channel: {}", - Thread.currentThread().getId(), selectionKey.channel()); - } catch (CancelledKeyException e) { - LOGGER.trace("Thread {} encountered cancelled channel: {}", - Thread.currentThread().getId(), selectionKey.channel()); - } - } - - private void connectionRead(SocketChannel socketChannel) { - Peer peer = getPeerFromChannel(socketChannel); - if (peer == null) { - return; - } - - try { - peer.readChannel(); - - LOGGER.trace("Thread {} re-registering OP_READ interestOps on channel: {}", - Thread.currentThread().getId(), socketChannel); - socketChannel.register(channelSelector, SelectionKey.OP_READ); - } catch (IOException e) { - if (e.getMessage() != null && e.getMessage().toLowerCase().contains("connection reset")) { - peer.disconnect("Connection reset"); - return; - } - - LOGGER.trace("[{}] Network thread {} encountered I/O error: {}", peer.getPeerConnectionId(), - Thread.currentThread().getId(), e.getMessage(), e); - peer.disconnect("I/O error"); - } - } + return new BroadcastTask(); } private Task maybeProduceChannelTask(boolean canBlock) throws InterruptedException { - final SelectionKey nextSelectionKey; - // Synchronization here to enforce thread-safety on channelIterator synchronized (channelSelector) { // anything to do? @@ -636,99 +566,45 @@ public class Network { } channelIterator = channelSelector.selectedKeys().iterator(); + LOGGER.trace("Thread {}, after {} select, channelIterator now {}", + Thread.currentThread().getId(), + canBlock ? "blocking": "non-blocking", + channelIterator); } - if (channelIterator.hasNext()) { - nextSelectionKey = channelIterator.next(); - channelIterator.remove(); - - if (nextSelectionKey.isReadable()) { - LOGGER.trace("Thread {} clearing all interestOps on channel: {}", - Thread.currentThread().getId(), nextSelectionKey.channel()); - nextSelectionKey.interestOps(0); - } - } else { - nextSelectionKey = null; + if (!channelIterator.hasNext()) { channelIterator = null; // Nothing to do so reset iterator to cause new select + + LOGGER.trace("Thread {}, channelIterator now null", Thread.currentThread().getId()); + return null; } - LOGGER.trace("Thread {}, nextSelectionKey {}, channelIterator now {}", - Thread.currentThread().getId(), nextSelectionKey, channelIterator); + final SelectionKey nextSelectionKey = channelIterator.next(); + channelIterator.remove(); + + LOGGER.trace("Thread {}, nextSelectionKey {}", Thread.currentThread().getId(), nextSelectionKey); + + if (nextSelectionKey.isReadable()) { + clearInterestOps(nextSelectionKey, SelectionKey.OP_READ); + return new ChannelReadTask(nextSelectionKey); + } + + if (nextSelectionKey.isWritable()) { + clearInterestOps(nextSelectionKey, SelectionKey.OP_WRITE); + return new ChannelWriteTask(nextSelectionKey); + } + + if (nextSelectionKey.isAcceptable()) { + clearInterestOps(nextSelectionKey, SelectionKey.OP_ACCEPT); + return new ChannelAcceptTask(nextSelectionKey); + } } - if (nextSelectionKey == null) { - return null; - } - - return new ChannelTask(nextSelectionKey); + return null; } } - private void acceptConnection(ServerSocketChannel serverSocketChannel) throws InterruptedException { - SocketChannel socketChannel; - - try { - if (getImmutableConnectedPeers().size() >= maxPeers) { - // We have enough peers - if (serverSelectionKey.interestOps() != 0) { - LOGGER.debug("Ignoring pending incoming connections because the server is full"); - serverSelectionKey.interestOps(0); - } - return; - } - - socketChannel = serverSocketChannel.accept(); - } catch (IOException e) { - return; - } - - // No connection actually accepted? - if (socketChannel == null) { - return; - } - PeerAddress address = PeerAddress.fromSocket(socketChannel.socket()); - List fixedNetwork = Settings.getInstance().getFixedNetwork(); - if (fixedNetwork != null && !fixedNetwork.isEmpty() && ipNotInFixedList(address, fixedNetwork)) { - try { - LOGGER.debug("Connection discarded from peer {} as not in the fixed network list", address); - socketChannel.close(); - } catch (IOException e) { - // IGNORE - } - return; - } - - final Long now = NTP.getTime(); - Peer newPeer; - - try { - if (now == null) { - LOGGER.debug("Connection discarded from peer {} due to lack of NTP sync", address); - socketChannel.close(); - return; - } - - LOGGER.debug("Connection accepted from peer {}", address); - - newPeer = new Peer(socketChannel, channelSelector); - this.addConnectedPeer(newPeer); - - } catch (IOException e) { - if (socketChannel.isOpen()) { - try { - LOGGER.debug("Connection failed from peer {} while connecting/closing", address); - socketChannel.close(); - } catch (IOException ce) { - // Couldn't close? - } - } - return; - } - - this.onPeerReady(newPeer); - } - - private boolean ipNotInFixedList(PeerAddress address, List fixedNetwork) { + public boolean ipNotInFixedList(PeerAddress address, List fixedNetwork) { for (String ipAddress : fixedNetwork) { String[] bits = ipAddress.split(":"); if (bits.length >= 1 && bits.length <= 2 && address.getHost().equals(bits[0])) { @@ -764,8 +640,9 @@ public class Network { peers.removeIf(isConnectedPeer); // Don't consider already connected peers (resolved address match) - // XXX This might be too slow if we end up waiting a long time for hostnames to resolve via DNS - peers.removeIf(isResolvedAsConnectedPeer); + // Disabled because this might be too slow if we end up waiting a long time for hostnames to resolve via DNS + // Which is ok because duplicate connections to the same peer are handled during handshaking + // peers.removeIf(isResolvedAsConnectedPeer); this.checkLongestConnection(now); @@ -795,12 +672,12 @@ public class Network { } } - private boolean connectPeer(Peer newPeer) throws InterruptedException { + public boolean connectPeer(Peer newPeer) throws InterruptedException { // NOT CORRECT: if (getImmutableConnectedPeers().size() >= minOutboundPeers) return false; - SocketChannel socketChannel = newPeer.connect(this.channelSelector); + SocketChannel socketChannel = newPeer.connect(); if (socketChannel == null) { return false; } @@ -815,7 +692,7 @@ public class Network { return true; } - private Peer getPeerFromChannel(SocketChannel socketChannel) { + public Peer getPeerFromChannel(SocketChannel socketChannel) { for (Peer peer : this.getImmutableConnectedPeers()) { if (peer.getSocketChannel() == socketChannel) { return peer; @@ -848,6 +725,69 @@ public class Network { nextDisconnectionCheck = now + DISCONNECTION_CHECK_INTERVAL; } + // SocketChannel interest-ops manipulations + + private static final String[] OP_NAMES = new String[SelectionKey.OP_ACCEPT * 2]; + static { + for (int i = 0; i < OP_NAMES.length; i++) { + StringJoiner joiner = new StringJoiner(","); + + if ((i & SelectionKey.OP_READ) != 0) joiner.add("OP_READ"); + if ((i & SelectionKey.OP_WRITE) != 0) joiner.add("OP_WRITE"); + if ((i & SelectionKey.OP_CONNECT) != 0) joiner.add("OP_CONNECT"); + if ((i & SelectionKey.OP_ACCEPT) != 0) joiner.add("OP_ACCEPT"); + + OP_NAMES[i] = joiner.toString(); + } + } + + public void clearInterestOps(SelectableChannel socketChannel, int interestOps) { + SelectionKey selectionKey = socketChannel.keyFor(channelSelector); + if (selectionKey == null) + return; + + clearInterestOps(selectionKey, interestOps); + } + + private void clearInterestOps(SelectionKey selectionKey, int interestOps) { + if (!selectionKey.channel().isOpen()) + return; + + LOGGER.trace("Thread {} clearing {} interest-ops on channel: {}", + Thread.currentThread().getId(), + OP_NAMES[interestOps], + selectionKey.channel()); + + selectionKey.interestOpsAnd(~interestOps); + } + + public void setInterestOps(SelectableChannel socketChannel, int interestOps) { + SelectionKey selectionKey = socketChannel.keyFor(channelSelector); + if (selectionKey == null) { + try { + selectionKey = socketChannel.register(this.channelSelector, interestOps); + } catch (ClosedChannelException e) { + // Channel already closed so ignore + return; + } + // Fall-through to allow logging + } + + setInterestOps(selectionKey, interestOps); + } + + private void setInterestOps(SelectionKey selectionKey, int interestOps) { + if (!selectionKey.channel().isOpen()) + return; + + LOGGER.trace("Thread {} setting {} interest-ops on channel: {}", + Thread.currentThread().getId(), + OP_NAMES[interestOps], + selectionKey.channel()); + + selectionKey.interestOpsOr(interestOps); + } + // Peer callbacks protected void wakeupChannelSelector() { @@ -887,7 +827,7 @@ public class Network { if (getImmutableConnectedPeers().size() < maxPeers - 1 && (serverSelectionKey.interestOps() & SelectionKey.OP_ACCEPT) == 0) { try { LOGGER.debug("Re-enabling accepting incoming connections because the server is not longer full"); - serverSelectionKey.interestOps(SelectionKey.OP_ACCEPT); + setInterestOps(serverSelectionKey, SelectionKey.OP_ACCEPT); } catch (CancelledKeyException e) { LOGGER.error("Failed to re-enable accepting of incoming connections: {}", e.getMessage()); } diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index b1dd0ef5..a755632d 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -12,24 +12,20 @@ import org.qortal.data.network.PeerData; import org.qortal.network.message.ChallengeMessage; import org.qortal.network.message.Message; import org.qortal.network.message.MessageException; -import org.qortal.network.message.MessageType; -import org.qortal.network.message.PingMessage; +import org.qortal.network.task.MessageTask; +import org.qortal.network.task.PingTask; import org.qortal.settings.Settings; -import org.qortal.utils.ExecuteProduceConsume; +import org.qortal.utils.ExecuteProduceConsume.Task; import org.qortal.utils.NTP; import java.io.IOException; import java.net.*; import java.nio.ByteBuffer; import java.nio.channels.SelectionKey; -import java.nio.channels.Selector; import java.nio.channels.SocketChannel; import java.security.SecureRandom; import java.util.*; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -98,7 +94,7 @@ public class Peer { /** * When last PING message was sent, or null if pings not started yet. */ - private Long lastPingSent; + private Long lastPingSent = null; byte[] ourChallenge; @@ -160,10 +156,10 @@ public class Peer { /** * Construct Peer using existing, connected socket */ - public Peer(SocketChannel socketChannel, Selector channelSelector) throws IOException { + public Peer(SocketChannel socketChannel) throws IOException { this.isOutbound = false; this.socketChannel = socketChannel; - sharedSetup(channelSelector); + sharedSetup(); this.resolvedAddress = ((InetSocketAddress) socketChannel.socket().getRemoteSocketAddress()); this.isLocal = isAddressLocal(this.resolvedAddress.getAddress()); @@ -276,7 +272,7 @@ public class Peer { } } - protected void setLastPing(long lastPing) { + public void setLastPing(long lastPing) { synchronized (this.peerInfoLock) { this.lastPing = lastPing; } @@ -396,13 +392,13 @@ public class Peer { // Processing - private void sharedSetup(Selector channelSelector) throws IOException { + private void sharedSetup() throws IOException { this.connectionTimestamp = NTP.getTime(); this.socketChannel.setOption(StandardSocketOptions.TCP_NODELAY, true); this.socketChannel.configureBlocking(false); - this.socketChannel.register(channelSelector, SelectionKey.OP_READ); + Network.getInstance().setInterestOps(this.socketChannel, SelectionKey.OP_READ); this.byteBuffer = null; // Defer allocation to when we need it, to save memory. Sorry GC! - this.replyQueues = Collections.synchronizedMap(new HashMap>()); + this.replyQueues = new ConcurrentHashMap<>(); this.pendingMessages = new LinkedBlockingQueue<>(); Random random = new SecureRandom(); @@ -410,7 +406,7 @@ public class Peer { random.nextBytes(this.ourChallenge); } - public SocketChannel connect(Selector channelSelector) { + public SocketChannel connect() { LOGGER.trace("[{}] Connecting to peer {}", this.peerConnectionId, this); try { @@ -432,7 +428,7 @@ public class Peer { try { LOGGER.debug("[{}] Connected to peer {}", this.peerConnectionId, this); - sharedSetup(channelSelector); + sharedSetup(); return socketChannel; } catch (IOException e) { LOGGER.trace("[{}] Post-connection setup failed, peer {}", this.peerConnectionId, this); @@ -450,7 +446,7 @@ public class Peer { * * @throws IOException If this channel is not yet connected */ - protected void readChannel() throws IOException { + public void readChannel() throws IOException { synchronized (this.byteBufferLock) { while (true) { if (!this.socketChannel.isOpen() || this.socketChannel.socket().isClosed()) { @@ -556,7 +552,16 @@ public class Peer { } } - protected ExecuteProduceConsume.Task getMessageTask() { + /** Maybe send some pending outgoing messages. + * + * @return true if more data is pending to be sent + */ + public boolean writeChannel() throws IOException { + // TODO + return false; + } + + protected Task getMessageTask() { /* * If we are still handshaking and there is a message yet to be processed then * don't produce another message task. This allows us to process handshake @@ -580,7 +585,7 @@ public class Peer { } // Return a task to process message in queue - return () -> Network.getInstance().onMessage(this, nextMessage); + return new MessageTask(this, nextMessage); } /** @@ -720,7 +725,7 @@ public class Peer { this.lastPingSent = NTP.getTime(); } - protected ExecuteProduceConsume.Task getPingTask(Long now) { + protected Task getPingTask(Long now) { // Pings not enabled yet? if (now == null || this.lastPingSent == null) { return null; @@ -734,19 +739,7 @@ public class Peer { // Not strictly true, but prevents this peer from being immediately chosen again this.lastPingSent = now; - return () -> { - PingMessage pingMessage = new PingMessage(); - Message message = this.getResponse(pingMessage); - - if (message == null || message.getType() != MessageType.PING) { - LOGGER.debug("[{}] Didn't receive reply from {} for PING ID {}", this.peerConnectionId, this, - pingMessage.getId()); - this.disconnect("no ping received"); - return; - } - - this.setLastPing(NTP.getTime() - now); - }; + return new PingTask(this, now); } public void disconnect(String reason) { diff --git a/src/main/java/org/qortal/network/task/BroadcastTask.java b/src/main/java/org/qortal/network/task/BroadcastTask.java new file mode 100644 index 00000000..5714ebf6 --- /dev/null +++ b/src/main/java/org/qortal/network/task/BroadcastTask.java @@ -0,0 +1,22 @@ +package org.qortal.network.task; + +import org.qortal.controller.Controller; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.Message; +import org.qortal.utils.ExecuteProduceConsume.Task; + +public class BroadcastTask implements Task { + public BroadcastTask() { + } + + @Override + public String getName() { + return "BroadcastTask"; + } + + @Override + public void perform() throws InterruptedException { + Controller.getInstance().doNetworkBroadcast(); + } +} diff --git a/src/main/java/org/qortal/network/task/ChannelAcceptTask.java b/src/main/java/org/qortal/network/task/ChannelAcceptTask.java new file mode 100644 index 00000000..b98a881a --- /dev/null +++ b/src/main/java/org/qortal/network/task/ChannelAcceptTask.java @@ -0,0 +1,99 @@ +package org.qortal.network.task; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.PeerAddress; +import org.qortal.settings.Settings; +import org.qortal.utils.ExecuteProduceConsume.Task; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.nio.channels.SelectionKey; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.util.List; + +public class ChannelAcceptTask implements Task { + private static final Logger LOGGER = LogManager.getLogger(ChannelAcceptTask.class); + + private final SelectionKey serverSelectionKey; + private final ServerSocketChannel serverSocketChannel; + + public ChannelAcceptTask(SelectionKey selectionKey) { + this.serverSelectionKey = selectionKey; + this.serverSocketChannel = (ServerSocketChannel) this.serverSelectionKey.channel(); + } + + @Override + public String getName() { + return "ChannelAcceptTask"; + } + + @Override + public void perform() throws InterruptedException { + Network network = Network.getInstance(); + SocketChannel socketChannel; + + try { + if (network.getImmutableConnectedPeers().size() >= network.getMaxPeers()) { + // We have enough peers + LOGGER.debug("Ignoring pending incoming connections because the server is full"); + return; + } + + socketChannel = serverSocketChannel.accept(); + + network.setInterestOps(serverSocketChannel, SelectionKey.OP_ACCEPT); + } catch (IOException e) { + return; + } + + // No connection actually accepted? + if (socketChannel == null) { + return; + } + + PeerAddress address = PeerAddress.fromSocket(socketChannel.socket()); + List fixedNetwork = Settings.getInstance().getFixedNetwork(); + if (fixedNetwork != null && !fixedNetwork.isEmpty() && network.ipNotInFixedList(address, fixedNetwork)) { + try { + LOGGER.debug("Connection discarded from peer {} as not in the fixed network list", address); + socketChannel.close(); + } catch (IOException e) { + // IGNORE + } + return; + } + + final Long now = NTP.getTime(); + Peer newPeer; + + try { + if (now == null) { + LOGGER.debug("Connection discarded from peer {} due to lack of NTP sync", address); + socketChannel.close(); + return; + } + + LOGGER.debug("Connection accepted from peer {}", address); + + newPeer = new Peer(socketChannel); + network.addConnectedPeer(newPeer); + + } catch (IOException e) { + if (socketChannel.isOpen()) { + try { + LOGGER.debug("Connection failed from peer {} while connecting/closing", address); + socketChannel.close(); + } catch (IOException ce) { + // Couldn't close? + } + } + return; + } + + network.onPeerReady(newPeer); + } +} diff --git a/src/main/java/org/qortal/network/task/ChannelReadTask.java b/src/main/java/org/qortal/network/task/ChannelReadTask.java new file mode 100644 index 00000000..ad190ef2 --- /dev/null +++ b/src/main/java/org/qortal/network/task/ChannelReadTask.java @@ -0,0 +1,55 @@ +package org.qortal.network.task; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.utils.ExecuteProduceConsume.Task; + +import java.io.IOException; +import java.nio.channels.SelectionKey; +import java.nio.channels.SocketChannel; + +public class ChannelReadTask implements Task { + private static final Logger LOGGER = LogManager.getLogger(ChannelReadTask.class); + + private final SelectionKey selectionKey; + private final SocketChannel socketChannel; + private final Peer peer; + private final String name; + + public ChannelReadTask(SelectionKey selectionKey) { + this.selectionKey = selectionKey; + this.socketChannel = (SocketChannel) this.selectionKey.channel(); + this.peer = Network.getInstance().getPeerFromChannel(this.socketChannel); + this.name = "ChannelReadTask::" + peer; + } + + @Override + public String getName() { + return name; + } + + @Override + public void perform() throws InterruptedException { + if (peer == null) { + return; + } + + try { + peer.readChannel(); + + Network.getInstance().setInterestOps(socketChannel, SelectionKey.OP_READ); + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().toLowerCase().contains("connection reset")) { + peer.disconnect("Connection reset"); + return; + } + + LOGGER.trace("[{}] Network thread {} encountered I/O error: {}", peer.getPeerConnectionId(), + Thread.currentThread().getId(), e.getMessage(), e); + peer.disconnect("I/O error"); + } + } +} diff --git a/src/main/java/org/qortal/network/task/ChannelWriteTask.java b/src/main/java/org/qortal/network/task/ChannelWriteTask.java new file mode 100644 index 00000000..757fa01d --- /dev/null +++ b/src/main/java/org/qortal/network/task/ChannelWriteTask.java @@ -0,0 +1,56 @@ +package org.qortal.network.task; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.utils.ExecuteProduceConsume.Task; + +import java.io.IOException; +import java.nio.channels.SelectionKey; +import java.nio.channels.SocketChannel; + +public class ChannelWriteTask implements Task { + private static final Logger LOGGER = LogManager.getLogger(ChannelWriteTask.class); + + private final SelectionKey selectionKey; + private final SocketChannel socketChannel; + private final Peer peer; + private final String name; + + public ChannelWriteTask(SelectionKey selectionKey) { + this.selectionKey = selectionKey; + this.socketChannel = (SocketChannel) this.selectionKey.channel(); + this.peer = Network.getInstance().getPeerFromChannel(this.socketChannel); + this.name = "ChannelWriteTask::" + peer; + } + + @Override + public String getName() { + return name; + } + + @Override + public void perform() throws InterruptedException { + if (peer == null) { + return; + } + + try { + boolean isMoreDataPending = peer.writeChannel(); + + if (isMoreDataPending) { + Network.getInstance().setInterestOps(socketChannel, SelectionKey.OP_WRITE); + } + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().toLowerCase().contains("connection reset")) { + peer.disconnect("Connection reset"); + return; + } + + LOGGER.trace("[{}] Network thread {} encountered I/O error: {}", peer.getPeerConnectionId(), + Thread.currentThread().getId(), e.getMessage(), e); + peer.disconnect("I/O error"); + } + } +} diff --git a/src/main/java/org/qortal/network/task/MessageTask.java b/src/main/java/org/qortal/network/task/MessageTask.java new file mode 100644 index 00000000..c1907b62 --- /dev/null +++ b/src/main/java/org/qortal/network/task/MessageTask.java @@ -0,0 +1,28 @@ +package org.qortal.network.task; + +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.Message; +import org.qortal.utils.ExecuteProduceConsume.Task; + +public class MessageTask implements Task { + private final Peer peer; + private final Message nextMessage; + private final String name; + + public MessageTask(Peer peer, Message nextMessage) { + this.peer = peer; + this.nextMessage = nextMessage; + this.name = "MessageTask::" + peer + "::" + nextMessage.getType(); + } + + @Override + public String getName() { + return name; + } + + @Override + public void perform() throws InterruptedException { + Network.getInstance().onMessage(peer, nextMessage); + } +} diff --git a/src/main/java/org/qortal/network/task/PeerConnectTask.java b/src/main/java/org/qortal/network/task/PeerConnectTask.java new file mode 100644 index 00000000..759cabce --- /dev/null +++ b/src/main/java/org/qortal/network/task/PeerConnectTask.java @@ -0,0 +1,33 @@ +package org.qortal.network.task; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.Message; +import org.qortal.network.message.MessageType; +import org.qortal.network.message.PingMessage; +import org.qortal.utils.ExecuteProduceConsume.Task; +import org.qortal.utils.NTP; + +public class PeerConnectTask implements Task { + private static final Logger LOGGER = LogManager.getLogger(PeerConnectTask.class); + + private final Peer peer; + private final String name; + + public PeerConnectTask(Peer peer) { + this.peer = peer; + this.name = "PeerConnectTask::" + peer; + } + + @Override + public String getName() { + return name; + } + + @Override + public void perform() throws InterruptedException { + Network.getInstance().connectPeer(peer); + } +} diff --git a/src/main/java/org/qortal/network/task/PingTask.java b/src/main/java/org/qortal/network/task/PingTask.java new file mode 100644 index 00000000..f47ecd32 --- /dev/null +++ b/src/main/java/org/qortal/network/task/PingTask.java @@ -0,0 +1,44 @@ +package org.qortal.network.task; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.network.Peer; +import org.qortal.network.message.Message; +import org.qortal.network.message.MessageType; +import org.qortal.network.message.PingMessage; +import org.qortal.utils.ExecuteProduceConsume.Task; +import org.qortal.utils.NTP; + +public class PingTask implements Task { + private static final Logger LOGGER = LogManager.getLogger(PingTask.class); + + private final Peer peer; + private final Long now; + private final String name; + + public PingTask(Peer peer, Long now) { + this.peer = peer; + this.now = now; + this.name = "PingTask::" + peer; + } + + @Override + public String getName() { + return name; + } + + @Override + public void perform() throws InterruptedException { + PingMessage pingMessage = new PingMessage(); + Message message = peer.getResponse(pingMessage); + + if (message == null || message.getType() != MessageType.PING) { + LOGGER.debug("[{}] Didn't receive reply from {} for PING ID {}", + peer.getPeerConnectionId(), peer, pingMessage.getId()); + peer.disconnect("no ping received"); + return; + } + + peer.setLastPing(NTP.getTime() - now); + } +} diff --git a/src/main/java/org/qortal/utils/ExecuteProduceConsume.java b/src/main/java/org/qortal/utils/ExecuteProduceConsume.java index d8e4dbf3..223d0e93 100644 --- a/src/main/java/org/qortal/utils/ExecuteProduceConsume.java +++ b/src/main/java/org/qortal/utils/ExecuteProduceConsume.java @@ -97,9 +97,9 @@ public abstract class ExecuteProduceConsume implements Runnable { */ protected abstract Task produceTask(boolean canBlock) throws InterruptedException; - @FunctionalInterface public interface Task { - public abstract void perform() throws InterruptedException; + String getName(); + void perform() throws InterruptedException; } @Override @@ -152,7 +152,7 @@ public abstract class ExecuteProduceConsume implements Runnable { if (this.logger.isDebugEnabled()) { final long productionPeriod = System.currentTimeMillis() - beforeProduce; - taskType = task == null ? "no task" : task.getClass().getCanonicalName(); + taskType = task == null ? "no task" : task.getName(); this.logger.debug(() -> String.format("[%d] produced [%s] in %dms [canBlock: %b]", Thread.currentThread().getId(), diff --git a/src/test/java/org/qortal/test/EPCTests.java b/src/test/java/org/qortal/test/EPCTests.java index fe48af24..1a41b75d 100644 --- a/src/test/java/org/qortal/test/EPCTests.java +++ b/src/test/java/org/qortal/test/EPCTests.java @@ -13,9 +13,25 @@ import org.junit.Test; import org.qortal.utils.ExecuteProduceConsume; import org.qortal.utils.ExecuteProduceConsume.StatsSnapshot; +import static org.junit.Assert.fail; + public class EPCTests { - class RandomEPC extends ExecuteProduceConsume { + static class SleepTask implements ExecuteProduceConsume.Task { + private static final Random RANDOM = new Random(); + + @Override + public String getName() { + return "SleepTask"; + } + + @Override + public void perform() throws InterruptedException { + Thread.sleep(RANDOM.nextInt(500) + 100); + } + } + + static class RandomEPC extends ExecuteProduceConsume { private final int TASK_PERCENT; private final int PAUSE_PERCENT; @@ -37,9 +53,7 @@ public class EPCTests { // Sometimes produce a task if (percent < TASK_PERCENT) { - return () -> { - Thread.sleep(random.nextInt(500) + 100); - }; + return new SleepTask(); } else { // If we don't produce a task, then maybe simulate a pause until work arrives if (canIdle && percent < PAUSE_PERCENT) @@ -50,45 +64,6 @@ public class EPCTests { } } - private void testEPC(ExecuteProduceConsume testEPC) throws InterruptedException { - final int runTime = 60; // seconds - System.out.println(String.format("Testing EPC for %s seconds:", runTime)); - - final long start = System.currentTimeMillis(); - testEPC.start(); - - // Status reports every second (bar waiting for synchronization) - ScheduledExecutorService statusExecutor = Executors.newSingleThreadScheduledExecutor(); - - statusExecutor.scheduleAtFixedRate(() -> { - final StatsSnapshot snapshot = testEPC.getStatsSnapshot(); - final long seconds = (System.currentTimeMillis() - start) / 1000L; - System.out.print(String.format("After %d second%s, ", seconds, (seconds != 1 ? "s" : ""))); - printSnapshot(snapshot); - }, 1L, 1L, TimeUnit.SECONDS); - - // Let it run for a minute - Thread.sleep(runTime * 1000L); - statusExecutor.shutdownNow(); - - final long before = System.currentTimeMillis(); - testEPC.shutdown(30 * 1000); - final long after = System.currentTimeMillis(); - - System.out.println(String.format("Shutdown took %d milliseconds", after - before)); - - final StatsSnapshot snapshot = testEPC.getStatsSnapshot(); - System.out.print("After shutdown, "); - printSnapshot(snapshot); - } - - private void printSnapshot(final StatsSnapshot snapshot) { - System.out.println(String.format("threads: %d active (%d max, %d exhaustion%s), tasks: %d produced / %d consumed", - snapshot.activeThreadCount, snapshot.greatestActiveThreadCount, - snapshot.spawnFailures, (snapshot.spawnFailures != 1 ? "s": ""), - snapshot.tasksProduced, snapshot.tasksConsumed)); - } - @Test public void testRandomEPC() throws InterruptedException { final int TASK_PERCENT = 25; // Produce a task this % of the time @@ -131,18 +106,39 @@ public class EPCTests { final int MAX_PEERS = 20; - final List lastPings = new ArrayList<>(Collections.nCopies(MAX_PEERS, System.currentTimeMillis())); + final List lastPingProduced = new ArrayList<>(Collections.nCopies(MAX_PEERS, System.currentTimeMillis())); class PingTask implements ExecuteProduceConsume.Task { private final int peerIndex; + private final long lastPing; + private final long productionTimestamp; + private final String name; - public PingTask(int peerIndex) { + public PingTask(int peerIndex, long lastPing, long productionTimestamp) { this.peerIndex = peerIndex; + this.lastPing = lastPing; + this.productionTimestamp = productionTimestamp; + this.name = "PingTask::[" + this.peerIndex + "]"; + } + + @Override + public String getName() { + return name; } @Override public void perform() throws InterruptedException { - System.out.println("Pinging peer " + peerIndex); + long now = System.currentTimeMillis(); + + System.out.println(String.format("Pinging peer %d after post-production delay of %dms and ping interval of %dms", + peerIndex, + now - productionTimestamp, + now - lastPing + )); + + long threshold = now - PING_INTERVAL - PRODUCER_SLEEP_TIME; + if (lastPing < threshold) + fail("excessive peer ping interval for peer " + peerIndex); // At least half the worst case ping round-trip Random random = new Random(); @@ -155,32 +151,73 @@ public class EPCTests { class PingEPC extends ExecuteProduceConsume { @Override protected Task produceTask(boolean canIdle) throws InterruptedException { - // If we can idle, then we do, to simulate worst case - if (canIdle) - Thread.sleep(PRODUCER_SLEEP_TIME); - // Is there a peer that needs a ping? final long now = System.currentTimeMillis(); - synchronized (lastPings) { - for (int peerIndex = 0; peerIndex < lastPings.size(); ++peerIndex) { - long lastPing = lastPings.get(peerIndex); - - if (lastPing < now - PING_INTERVAL - PING_ROUND_TRIP_TIME - PRODUCER_SLEEP_TIME) - throw new RuntimeException("excessive peer ping interval for peer " + peerIndex); + synchronized (lastPingProduced) { + for (int peerIndex = 0; peerIndex < lastPingProduced.size(); ++peerIndex) { + long lastPing = lastPingProduced.get(peerIndex); if (lastPing < now - PING_INTERVAL) { - lastPings.set(peerIndex, System.currentTimeMillis()); - return new PingTask(peerIndex); + lastPingProduced.set(peerIndex, System.currentTimeMillis()); + return new PingTask(peerIndex, lastPing, now); } } } + // If we can idle, then we do, to simulate worst case + if (canIdle) + Thread.sleep(PRODUCER_SLEEP_TIME); + // No work to do return null; } } + System.out.println(String.format("Pings should start after %s seconds", PING_INTERVAL)); + testEPC(new PingEPC()); } + private void testEPC(ExecuteProduceConsume testEPC) throws InterruptedException { + final int runTime = 60; // seconds + System.out.println(String.format("Testing EPC for %s seconds:", runTime)); + + final long start = System.currentTimeMillis(); + + // Status reports every second (bar waiting for synchronization) + ScheduledExecutorService statusExecutor = Executors.newSingleThreadScheduledExecutor(); + + statusExecutor.scheduleAtFixedRate( + () -> { + final StatsSnapshot snapshot = testEPC.getStatsSnapshot(); + final long seconds = (System.currentTimeMillis() - start) / 1000L; + System.out.println(String.format("After %d second%s, %s", seconds, seconds != 1 ? "s" : "", formatSnapshot(snapshot))); + }, + 0L, 1L, TimeUnit.SECONDS + ); + + testEPC.start(); + + // Let it run for a minute + Thread.sleep(runTime * 1000L); + statusExecutor.shutdownNow(); + + final long before = System.currentTimeMillis(); + testEPC.shutdown(30 * 1000); + final long after = System.currentTimeMillis(); + + System.out.println(String.format("Shutdown took %d milliseconds", after - before)); + + final StatsSnapshot snapshot = testEPC.getStatsSnapshot(); + System.out.println("After shutdown, " + formatSnapshot(snapshot)); + } + + private String formatSnapshot(StatsSnapshot snapshot) { + return String.format("threads: %d active (%d max, %d exhaustion%s), tasks: %d produced / %d consumed", + snapshot.activeThreadCount, snapshot.greatestActiveThreadCount, + snapshot.spawnFailures, (snapshot.spawnFailures != 1 ? "s": ""), + snapshot.tasksProduced, snapshot.tasksConsumed + ); + } + } From 3505788d42c6812129623a809b5f90338275db3c Mon Sep 17 00:00:00 2001 From: catbref Date: Thu, 31 Mar 2022 20:52:23 +0100 Subject: [PATCH 08/14] Another chunk of improvements to networking / EPC. Instead of synchronizing/blocking in Peer.sendMessage(), we queue messages in a concurrent blocking TransferQueue, with timeout. In EPC, ChannelWriteTasks consume from TransferQueue, unblocking callers to Peer.sendMessage(). If a new message is to be sent, or socket output buffer is full, then OP_WRITE is used to wait for socket to become writable again. Only one ChannelWriteTask per peer can be active/pending at a time. Each ChannelWriteTask tries to send as much as it can in one go. Other minor tidy-ups. --- src/main/java/org/qortal/network/Network.java | 50 ++++--- src/main/java/org/qortal/network/Peer.java | 123 ++++++++++-------- .../network/task/ChannelAcceptTask.java | 6 +- .../qortal/network/task/ChannelReadTask.java | 12 +- .../qortal/network/task/ChannelWriteTask.java | 22 ++-- 5 files changed, 118 insertions(+), 95 deletions(-) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 3ee7af75..fff27390 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -33,6 +33,7 @@ import java.nio.channels.*; import java.security.SecureRandom; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; @@ -42,7 +43,6 @@ import java.util.stream.Collectors; // For managing peers public class Network { private static final Logger LOGGER = LogManager.getLogger(Network.class); - private static Network instance; private static final int LISTEN_BACKLOG = 5; /** @@ -124,14 +124,9 @@ public class Network { private Selector channelSelector; private ServerSocketChannel serverChannel; private SelectionKey serverSelectionKey; - private Iterator channelIterator = null; - - // volatile because value is updated inside any one of the EPC threads - private volatile long nextConnectTaskTimestamp = 0L; // ms - try first connect once NTP syncs + private final Set channelsPendingWrite = ConcurrentHashMap.newKeySet(); private final ExecutorService broadcastExecutor = Executors.newCachedThreadPool(); - // volatile because value is updated inside any one of the EPC threads - private volatile long nextBroadcastTimestamp = 0L; // ms - try first broadcast once NTP syncs private final Lock mergePeersLock = new ReentrantLock(); @@ -460,6 +455,11 @@ public class Network { class NetworkProcessor extends ExecuteProduceConsume { + private final AtomicLong nextConnectTaskTimestamp = new AtomicLong(0L); // ms - try first connect once NTP syncs + private final AtomicLong nextBroadcastTimestamp = new AtomicLong(0L); // ms - try first broadcast once NTP syncs + + private Iterator channelIterator = null; + NetworkProcessor(ExecutorService executor) { super(executor); } @@ -517,7 +517,7 @@ public class Network { } private Task maybeProduceConnectPeerTask(Long now) throws InterruptedException { - if (now == null || now < nextConnectTaskTimestamp) { + if (now == null || now < nextConnectTaskTimestamp.get()) { return null; } @@ -525,7 +525,7 @@ public class Network { return null; } - nextConnectTaskTimestamp = now + 1000L; + nextConnectTaskTimestamp.set(now + 1000L); Peer targetPeer = getConnectablePeer(now); if (targetPeer == null) { @@ -537,11 +537,11 @@ public class Network { } private Task maybeProduceBroadcastTask(Long now) { - if (now == null || now < nextBroadcastTimestamp) { + if (now == null || now < nextBroadcastTimestamp.get()) { return null; } - nextBroadcastTimestamp = now + BROADCAST_INTERVAL; + nextBroadcastTimestamp.set(now + BROADCAST_INTERVAL); return new BroadcastTask(); } @@ -584,19 +584,34 @@ public class Network { LOGGER.trace("Thread {}, nextSelectionKey {}", Thread.currentThread().getId(), nextSelectionKey); + SelectableChannel socketChannel = nextSelectionKey.channel(); + if (nextSelectionKey.isReadable()) { clearInterestOps(nextSelectionKey, SelectionKey.OP_READ); - return new ChannelReadTask(nextSelectionKey); + Peer peer = getPeerFromChannel((SocketChannel) socketChannel); + if (peer == null) + return null; + + return new ChannelReadTask((SocketChannel) socketChannel, peer); } if (nextSelectionKey.isWritable()) { clearInterestOps(nextSelectionKey, SelectionKey.OP_WRITE); - return new ChannelWriteTask(nextSelectionKey); + Peer peer = getPeerFromChannel((SocketChannel) socketChannel); + if (peer == null) + return null; + + // Any thread that queues a message to send can set OP_WRITE, + // but we only allow one pending/active ChannelWriteTask per Peer + if (!channelsPendingWrite.add(socketChannel)) + return null; + + return new ChannelWriteTask((SocketChannel) socketChannel, peer); } if (nextSelectionKey.isAcceptable()) { clearInterestOps(nextSelectionKey, SelectionKey.OP_ACCEPT); - return new ChannelAcceptTask(nextSelectionKey); + return new ChannelAcceptTask((ServerSocketChannel) socketChannel); } } @@ -788,7 +803,11 @@ public class Network { selectionKey.interestOpsOr(interestOps); } - // Peer callbacks + // Peer / Task callbacks + + public void notifyChannelNotWriting(SelectableChannel socketChannel) { + this.channelsPendingWrite.remove(socketChannel); + } protected void wakeupChannelSelector() { this.channelSelector.wakeup(); @@ -823,6 +842,7 @@ public class Network { } this.removeConnectedPeer(peer); + this.channelsPendingWrite.remove(peer.getSocketChannel()); if (getImmutableConnectedPeers().size() < maxPeers - 1 && (serverSelectionKey.interestOps() & SelectionKey.OP_ACCEPT) == 0) { try { diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index a755632d..9d29fc1f 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -44,9 +44,9 @@ public class Peer { private static final int RESPONSE_TIMEOUT = 3000; // ms /** - * Maximum time to wait for a peer to respond with blocks (ms) + * Maximum time to wait for a message to be added to sendQueue (ms) */ - public static final int FETCH_BLOCKS_TIMEOUT = 10000; + private static final int QUEUE_TIMEOUT = 1000; // ms /** * Interval between PING messages to a peer. (ms) @@ -67,10 +67,14 @@ public class Peer { private final UUID peerConnectionId = UUID.randomUUID(); private final Object byteBufferLock = new Object(); private ByteBuffer byteBuffer; - private Map> replyQueues; private LinkedBlockingQueue pendingMessages; + private TransferQueue sendQueue; + private ByteBuffer outputBuffer; + private String outputMessageType; + private int outputMessageId; + /** * True if we created connection to peer, false if we accepted incoming connection from peer. */ @@ -342,12 +346,6 @@ public class Peer { } } - protected void queueMessage(Message message) { - if (!this.pendingMessages.offer(message)) { - LOGGER.info("[{}] No room to queue message from peer {} - discarding", this.peerConnectionId, this); - } - } - public boolean isSyncInProgress() { return this.syncInProgress; } @@ -398,6 +396,7 @@ public class Peer { this.socketChannel.configureBlocking(false); Network.getInstance().setInterestOps(this.socketChannel, SelectionKey.OP_READ); this.byteBuffer = null; // Defer allocation to when we need it, to save memory. Sorry GC! + this.sendQueue = new LinkedTransferQueue<>(); this.replyQueues = new ConcurrentHashMap<>(); this.pendingMessages = new LinkedBlockingQueue<>(); @@ -557,8 +556,59 @@ public class Peer { * @return true if more data is pending to be sent */ public boolean writeChannel() throws IOException { - // TODO - return false; + // It is the responsibility of ChannelWriteTask's producer to produce only one call to writeChannel() at a time + + while (true) { + // If output byte buffer is null, fetch next message from queue (if any) + while (this.outputBuffer == null) { + Message message; + + try { + // Allow other thread time to add message to queue having raised OP_WRITE. + // Timeout is overkill but not excessive enough to clog up networking / EPC. + // This is to avoid race condition in sendMessageWithTimeout() below. + message = this.sendQueue.poll(QUEUE_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + // Shutdown situation + return false; + } + + // No message? No further work to be done + if (message == null) + return false; + + try { + this.outputBuffer = ByteBuffer.wrap(message.toBytes()); + this.outputMessageType = message.getType().name(); + this.outputMessageId = message.getId(); + + LOGGER.trace("[{}] Sending {} message with ID {} to peer {}", + this.peerConnectionId, this.outputMessageType, this.outputMessageId, this); + } catch (MessageException e) { + // Something went wrong converting message to bytes, so discard but allow another round + LOGGER.warn("[{}] Failed to send {} message with ID {} to peer {}: {}", this.peerConnectionId, + message.getType().name(), message.getId(), this, e.getMessage()); + } + } + + // If output byte buffer is not null, send from that + int bytesWritten = this.socketChannel.write(outputBuffer); + + LOGGER.trace("[{}] Sent {} bytes of {} message with ID {} to peer {} ({} total)", this.peerConnectionId, + bytesWritten, this.outputMessageType, this.outputMessageId, this, outputBuffer.limit()); + + // If we've sent 0 bytes then socket buffer is full so we need to wait until it's empty again + if (bytesWritten == 0) { + return true; + } + + // If we then exhaust the byte buffer, set it to null (otherwise loop and try to send more) + if (!this.outputBuffer.hasRemaining()) { + this.outputMessageType = null; + this.outputMessageId = 0; + this.outputBuffer = null; + } + } } protected Task getMessageTask() { @@ -610,54 +660,19 @@ public class Peer { } try { - // Send message - LOGGER.trace("[{}] Sending {} message with ID {} to peer {}", this.peerConnectionId, + // Queue message, to be picked up by ChannelWriteTask and then peer.writeChannel() + LOGGER.trace("[{}] Queuing {} message with ID {} to peer {}", this.peerConnectionId, message.getType().name(), message.getId(), this); - ByteBuffer outputBuffer = ByteBuffer.wrap(message.toBytes()); - - synchronized (this.socketChannel) { - final long sendStart = System.currentTimeMillis(); - long totalBytes = 0; - - while (outputBuffer.hasRemaining()) { - int bytesWritten = this.socketChannel.write(outputBuffer); - totalBytes += bytesWritten; - - LOGGER.trace("[{}] Sent {} bytes of {} message with ID {} to peer {} ({} total)", this.peerConnectionId, - bytesWritten, message.getType().name(), message.getId(), this, totalBytes); - - if (bytesWritten == 0) { - // Underlying socket's internal buffer probably full, - // so wait a short while for bytes to actually be transmitted over the wire - - /* - * NOSONAR squid:S2276 - we don't want to use this.socketChannel.wait() - * as this releases the lock held by synchronized() above - * and would allow another thread to send another message, - * potentially interleaving them on-the-wire, causing checksum failures - * and connection loss. - */ - Thread.sleep(100L); //NOSONAR squid:S2276 - - if (System.currentTimeMillis() - sendStart > timeout) { - // We've taken too long to send this message - return false; - } - } - } - } - } catch (MessageException e) { - LOGGER.warn("[{}] Failed to send {} message with ID {} to peer {}: {}", this.peerConnectionId, - message.getType().name(), message.getId(), this, e.getMessage()); - return false; - } catch (IOException | InterruptedException e) { + // Possible race condition: + // We set OP_WRITE, EPC creates ChannelWriteTask which calls Peer.writeChannel, writeChannel's poll() finds no message to send + // Avoided by poll-with-timeout in writeChannel() above. + Network.getInstance().setInterestOps(this.socketChannel, SelectionKey.OP_WRITE); + return this.sendQueue.tryTransfer(message, timeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { // Send failure return false; } - - // Sent OK - return true; } /** diff --git a/src/main/java/org/qortal/network/task/ChannelAcceptTask.java b/src/main/java/org/qortal/network/task/ChannelAcceptTask.java index b98a881a..3e2a3033 100644 --- a/src/main/java/org/qortal/network/task/ChannelAcceptTask.java +++ b/src/main/java/org/qortal/network/task/ChannelAcceptTask.java @@ -18,12 +18,10 @@ import java.util.List; public class ChannelAcceptTask implements Task { private static final Logger LOGGER = LogManager.getLogger(ChannelAcceptTask.class); - private final SelectionKey serverSelectionKey; private final ServerSocketChannel serverSocketChannel; - public ChannelAcceptTask(SelectionKey selectionKey) { - this.serverSelectionKey = selectionKey; - this.serverSocketChannel = (ServerSocketChannel) this.serverSelectionKey.channel(); + public ChannelAcceptTask(ServerSocketChannel serverSocketChannel) { + this.serverSocketChannel = serverSocketChannel; } @Override diff --git a/src/main/java/org/qortal/network/task/ChannelReadTask.java b/src/main/java/org/qortal/network/task/ChannelReadTask.java index ad190ef2..edd4e8c0 100644 --- a/src/main/java/org/qortal/network/task/ChannelReadTask.java +++ b/src/main/java/org/qortal/network/task/ChannelReadTask.java @@ -14,15 +14,13 @@ import java.nio.channels.SocketChannel; public class ChannelReadTask implements Task { private static final Logger LOGGER = LogManager.getLogger(ChannelReadTask.class); - private final SelectionKey selectionKey; private final SocketChannel socketChannel; private final Peer peer; private final String name; - public ChannelReadTask(SelectionKey selectionKey) { - this.selectionKey = selectionKey; - this.socketChannel = (SocketChannel) this.selectionKey.channel(); - this.peer = Network.getInstance().getPeerFromChannel(this.socketChannel); + public ChannelReadTask(SocketChannel socketChannel, Peer peer) { + this.socketChannel = socketChannel; + this.peer = peer; this.name = "ChannelReadTask::" + peer; } @@ -33,10 +31,6 @@ public class ChannelReadTask implements Task { @Override public void perform() throws InterruptedException { - if (peer == null) { - return; - } - try { peer.readChannel(); diff --git a/src/main/java/org/qortal/network/task/ChannelWriteTask.java b/src/main/java/org/qortal/network/task/ChannelWriteTask.java index 757fa01d..59bc557e 100644 --- a/src/main/java/org/qortal/network/task/ChannelWriteTask.java +++ b/src/main/java/org/qortal/network/task/ChannelWriteTask.java @@ -13,15 +13,13 @@ import java.nio.channels.SocketChannel; public class ChannelWriteTask implements Task { private static final Logger LOGGER = LogManager.getLogger(ChannelWriteTask.class); - private final SelectionKey selectionKey; private final SocketChannel socketChannel; private final Peer peer; private final String name; - public ChannelWriteTask(SelectionKey selectionKey) { - this.selectionKey = selectionKey; - this.socketChannel = (SocketChannel) this.selectionKey.channel(); - this.peer = Network.getInstance().getPeerFromChannel(this.socketChannel); + public ChannelWriteTask(SocketChannel socketChannel, Peer peer) { + this.socketChannel = socketChannel; + this.peer = peer; this.name = "ChannelWriteTask::" + peer; } @@ -32,16 +30,14 @@ public class ChannelWriteTask implements Task { @Override public void perform() throws InterruptedException { - if (peer == null) { - return; - } - try { - boolean isMoreDataPending = peer.writeChannel(); + boolean isSocketClogged = peer.writeChannel(); - if (isMoreDataPending) { - Network.getInstance().setInterestOps(socketChannel, SelectionKey.OP_WRITE); - } + // Tell Network that we've finished + Network.getInstance().notifyChannelNotWriting(socketChannel); + + if (isSocketClogged) + Network.getInstance().setInterestOps(this.socketChannel, SelectionKey.OP_WRITE); } catch (IOException e) { if (e.getMessage() != null && e.getMessage().toLowerCase().contains("connection reset")) { peer.disconnect("Connection reset"); From 8e09567221bd75bb288cc1f922fad089cfae930f Mon Sep 17 00:00:00 2001 From: catbref Date: Thu, 31 Mar 2022 21:23:23 +0100 Subject: [PATCH 09/14] EPC-fixed: avoiding some CancelledKeyExceptions --- src/main/java/org/qortal/network/Network.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index fff27390..715dbc9a 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -582,6 +582,10 @@ public class Network { final SelectionKey nextSelectionKey = channelIterator.next(); channelIterator.remove(); + // Just in case underlying socket channel already closed elsewhere, etc. + if (!nextSelectionKey.isValid()) + return null; + LOGGER.trace("Thread {}, nextSelectionKey {}", Thread.currentThread().getId(), nextSelectionKey); SelectableChannel socketChannel = nextSelectionKey.channel(); @@ -844,7 +848,9 @@ public class Network { this.removeConnectedPeer(peer); this.channelsPendingWrite.remove(peer.getSocketChannel()); - if (getImmutableConnectedPeers().size() < maxPeers - 1 && (serverSelectionKey.interestOps() & SelectionKey.OP_ACCEPT) == 0) { + if (getImmutableConnectedPeers().size() < maxPeers - 1 + && serverSelectionKey.isValid() + && (serverSelectionKey.interestOps() & SelectionKey.OP_ACCEPT) == 0) { try { LOGGER.debug("Re-enabling accepting incoming connections because the server is not longer full"); setInterestOps(serverSelectionKey, SelectionKey.OP_ACCEPT); From 22aa5c41b5d6e23fc016627c3656f5882747a129 Mon Sep 17 00:00:00 2001 From: catbref Date: Sat, 2 Apr 2022 15:51:00 +0100 Subject: [PATCH 10/14] WIP: EPC-fixes BlockMessage was broken because the repository 'connection' associated with the message's Block object was closed between message queuing and message sending. The fix was to serialize Message subclasses on construction, thus freeing reliance on objects passed into constructor. The serialized byte[] is held by the message between queuing and sending. This forces messages into one of two 'modes': outgoing or incoming. Outgoing messages contain serialized byte[] whereas incoming messages unpack a ByteBuffer into Message subclass fields. As a result, all network message types have been refactored in this way. More details in Message's class comment. A knock-on effect is that incoming messages cannot then be sent out - a new message needs to be constructed. Some changes needed to Arbitrary controller package classes in this respect. Bonus: Network no longer needs broadcast threads because 'broadcasting' is now simply the act of queuing a message for many peers. --- .../org/qortal/controller/Controller.java | 8 +- .../controller/TransactionImporter.java | 5 +- .../ArbitraryDataFileListManager.java | 29 ++-- .../arbitrary/ArbitraryDataFileManager.java | 25 +-- .../arbitrary/ArbitraryMetadataManager.java | 11 +- src/main/java/org/qortal/network/Network.java | 56 +------ src/main/java/org/qortal/network/Peer.java | 6 + .../message/ArbitraryDataFileListMessage.java | 144 +++++++----------- .../message/ArbitraryDataFileMessage.java | 60 +++----- .../network/message/ArbitraryDataMessage.java | 37 +++-- .../message/ArbitraryMetadataMessage.java | 56 +++---- .../message/ArbitrarySignaturesMessage.java | 44 +++--- .../qortal/network/message/BlockMessage.java | 45 +----- .../message/BlockSummariesMessage.java | 38 ++--- .../network/message/CachedBlockMessage.java | 55 +++---- .../network/message/ChallengeMessage.java | 36 ++--- .../GetArbitraryDataFileListMessage.java | 91 ++++++----- .../message/GetArbitraryDataFileMessage.java | 30 ++-- .../message/GetArbitraryDataMessage.java | 19 +-- .../message/GetArbitraryMetadataMessage.java | 54 ++++--- .../network/message/GetBlockMessage.java | 19 +-- .../message/GetBlockSummariesMessage.java | 30 ++-- .../message/GetOnlineAccountsMessage.java | 36 ++--- .../message/GetOnlineAccountsV2Message.java | 78 +++++----- .../network/message/GetPeersMessage.java | 9 +- .../message/GetSignaturesV2Message.java | 30 ++-- .../message/GetTradePresencesMessage.java | 84 +++++----- .../message/GetTransactionMessage.java | 19 +-- .../GetUnconfirmedTransactionsMessage.java | 9 +- .../network/message/GoodbyeMessage.java | 19 +-- .../network/message/HeightV2Message.java | 42 ++--- .../qortal/network/message/HelloMessage.java | 42 ++--- .../org/qortal/network/message/Message.java | 101 ++++++++---- .../message/OnlineAccountsMessage.java | 40 ++--- .../message/OnlineAccountsV2Message.java | 89 ++++++----- .../network/message/PeersV2Message.java | 60 ++++---- .../qortal/network/message/PingMessage.java | 9 +- .../qortal/network/message/PongMessage.java | 9 +- .../network/message/ResponseMessage.java | 36 ++--- .../network/message/SignaturesMessage.java | 30 ++-- .../message/TradePresencesMessage.java | 95 ++++++------ .../network/message/TransactionMessage.java | 15 +- .../message/TransactionSignaturesMessage.java | 30 ++-- 43 files changed, 827 insertions(+), 953 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index fcf6270f..d896e32f 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -58,6 +58,7 @@ import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.transform.TransformationException; import org.qortal.utils.*; public class Controller extends Thread { @@ -1236,7 +1237,7 @@ public class Controller extends Thread { this.stats.getBlockMessageStats.cacheHits.incrementAndGet(); // We need to duplicate it to prevent multiple threads setting ID on the same message - CachedBlockMessage clonedBlockMessage = cachedBlockMessage.cloneWithNewId(message.getId()); + CachedBlockMessage clonedBlockMessage = Message.cloneWithNewId(cachedBlockMessage, message.getId()); if (!peer.sendMessage(clonedBlockMessage)) peer.disconnect("failed to send block"); @@ -1295,7 +1296,6 @@ public class Controller extends Thread { CachedBlockMessage blockMessage = new CachedBlockMessage(block); blockMessage.setId(message.getId()); - // This call also causes the other needed data to be pulled in from repository if (!peer.sendMessage(blockMessage)) { peer.disconnect("failed to send block"); // Don't fall-through to caching because failure to send might be from failure to build message @@ -1309,7 +1309,9 @@ public class Controller extends Thread { this.blockMessageCache.put(ByteArray.wrap(blockData.getSignature()), blockMessage); } } catch (DataException e) { - LOGGER.error(String.format("Repository issue while send block %s to peer %s", Base58.encode(signature), peer), e); + LOGGER.error(String.format("Repository issue while sending block %s to peer %s", Base58.encode(signature), peer), e); + } catch (TransformationException e) { + LOGGER.error(String.format("Serialization issue while sending block %s to peer %s", Base58.encode(signature), peer), e); } } diff --git a/src/main/java/org/qortal/controller/TransactionImporter.java b/src/main/java/org/qortal/controller/TransactionImporter.java index 3514ea47..16fd3a59 100644 --- a/src/main/java/org/qortal/controller/TransactionImporter.java +++ b/src/main/java/org/qortal/controller/TransactionImporter.java @@ -12,6 +12,7 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.transaction.Transaction; +import org.qortal.transform.TransformationException; import org.qortal.utils.Base58; import org.qortal.utils.NTP; @@ -289,7 +290,9 @@ public class TransactionImporter extends Thread { if (!peer.sendMessage(transactionMessage)) peer.disconnect("failed to send transaction"); } catch (DataException e) { - LOGGER.error(String.format("Repository issue while send transaction %s to peer %s", Base58.encode(signature), peer), e); + LOGGER.error(String.format("Repository issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e); + } catch (TransformationException e) { + LOGGER.error(String.format("Serialization issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index e855171d..05a45425 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -511,18 +511,23 @@ public class ArbitraryDataFileListManager { // Bump requestHops if it exists if (requestHops != null) { - arbitraryDataFileListMessage.setRequestHops(++requestHops); + requestHops++; } + ArbitraryDataFileListMessage forwardArbitraryDataFileListMessage; + // Remove optional parameters if the requesting peer doesn't support it yet // A message with less statistical data is better than no message at all if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { - arbitraryDataFileListMessage.removeOptionalStats(); + forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + } else { + forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, + arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); } // Forward to requesting peer LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer); - if (!requestingPeer.sendMessage(arbitraryDataFileListMessage)) { + if (!requestingPeer.sendMessage(forwardArbitraryDataFileListMessage)) { requestingPeer.disconnect("failed to forward arbitrary data file list"); } } @@ -639,16 +644,19 @@ public class ArbitraryDataFileListManager { } String ourAddress = Network.getInstance().getOurExternalIpAddressAndPort(); - ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, - hashes, NTP.getTime(), 0, ourAddress, true); - arbitraryDataFileListMessage.setId(message.getId()); + ArbitraryDataFileListMessage arbitraryDataFileListMessage; // Remove optional parameters if the requesting peer doesn't support it yet // A message with less statistical data is better than no message at all if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { - arbitraryDataFileListMessage.removeOptionalStats(); + arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + } else { + arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, + hashes, NTP.getTime(), 0, ourAddress, true); } + arbitraryDataFileListMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryDataFileListMessage)) { LOGGER.debug("Couldn't send list of hashes"); peer.disconnect("failed to send list of hashes"); @@ -670,8 +678,7 @@ public class ArbitraryDataFileListManager { // In relay mode - so ask our other peers if they have it long requestTime = getArbitraryDataFileListMessage.getRequestTime(); - int requestHops = getArbitraryDataFileListMessage.getRequestHops(); - getArbitraryDataFileListMessage.setRequestHops(++requestHops); + int requestHops = getArbitraryDataFileListMessage.getRequestHops() + 1; long totalRequestTime = now - requestTime; if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { @@ -679,11 +686,13 @@ public class ArbitraryDataFileListManager { if (requestHops < RELAY_REQUEST_MAX_HOPS) { // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast + Message relayGetArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, requestingPeer); + LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); Network.getInstance().broadcast( broadcastPeer -> broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) - ? null : getArbitraryDataFileListMessage); + ? null : relayGetArbitraryDataFileListMessage); } else { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 977f6215..d81b8145 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -186,7 +186,7 @@ public class ArbitraryDataFileManager extends Thread { ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature); boolean fileAlreadyExists = existingFile.exists(); String hash58 = Base58.encode(hash); - Message message = null; + ArbitraryDataFileMessage arbitraryDataFileMessage; // Fetch the file if it doesn't exist locally if (!fileAlreadyExists) { @@ -194,10 +194,11 @@ public class ArbitraryDataFileManager extends Thread { arbitraryDataFileRequests.put(hash58, NTP.getTime()); Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash); + Message response = null; try { - message = peer.getResponseWithTimeout(getArbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT); + response = peer.getResponseWithTimeout(getArbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT); } catch (InterruptedException e) { - // Will return below due to null message + // Will return below due to null response } arbitraryDataFileRequests.remove(hash58); LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58)); @@ -205,22 +206,24 @@ public class ArbitraryDataFileManager extends Thread { // We may need to remove the file list request, if we have all the files for this transaction this.handleFileListRequests(signature); - if (message == null) { - LOGGER.debug("Received null message from peer {}", peer); + if (response == null) { + LOGGER.debug("Received null response from peer {}", peer); return null; } - if (message.getType() != MessageType.ARBITRARY_DATA_FILE) { - LOGGER.debug("Received message with invalid type: {} from peer {}", message.getType(), peer); + if (response.getType() != MessageType.ARBITRARY_DATA_FILE) { + LOGGER.debug("Received response with invalid type: {} from peer {}", response.getType(), peer); return null; } - } - else { + + ArbitraryDataFileMessage peersArbitraryDataFileMessage = (ArbitraryDataFileMessage) response; + arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, peersArbitraryDataFileMessage.getArbitraryDataFile()); + } else { LOGGER.debug(String.format("File hash %s already exists, so skipping the request", hash58)); + arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, existingFile); } - ArbitraryDataFileMessage arbitraryDataFileMessage = (ArbitraryDataFileMessage) message; // We might want to forward the request to the peer that originally requested it - this.handleArbitraryDataFileForwarding(requestingPeer, message, originalMessage); + this.handleArbitraryDataFileForwarding(requestingPeer, arbitraryDataFileMessage, originalMessage); boolean isRelayRequest = (requestingPeer != null); if (isRelayRequest) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java index acc97f35..0903de60 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java @@ -338,9 +338,11 @@ public class ArbitraryMetadataManager { Peer requestingPeer = request.getB(); if (requestingPeer != null) { + ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile()); + // Forward to requesting peer LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer); - if (!requestingPeer.sendMessage(arbitraryMetadataMessage)) { + if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) { requestingPeer.disconnect("failed to forward arbitrary metadata"); } } @@ -423,8 +425,7 @@ public class ArbitraryMetadataManager { // In relay mode - so ask our other peers if they have it long requestTime = getArbitraryMetadataMessage.getRequestTime(); - int requestHops = getArbitraryMetadataMessage.getRequestHops(); - getArbitraryMetadataMessage.setRequestHops(++requestHops); + int requestHops = getArbitraryMetadataMessage.getRequestHops() + 1; long totalRequestTime = now - requestTime; if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { @@ -432,11 +433,13 @@ public class ArbitraryMetadataManager { if (requestHops < RELAY_REQUEST_MAX_HOPS) { // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast + Message relayGetArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, requestTime, requestHops); + LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); Network.getInstance().broadcast( broadcastPeer -> broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) - ? null : getArbitraryMetadataMessage); + ? null : relayGetArbitraryMetadataMessage); } else { diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 715dbc9a..a82107ec 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -126,8 +126,6 @@ public class Network { private SelectionKey serverSelectionKey; private final Set channelsPendingWrite = ConcurrentHashMap.newKeySet(); - private final ExecutorService broadcastExecutor = Executors.newCachedThreadPool(); - private final Lock mergePeersLock = new ReentrantLock(); private List ourExternalIpAddressHistory = new ArrayList<>(); @@ -1455,49 +1453,17 @@ public class Network { } public void broadcast(Function peerMessageBuilder) { - class Broadcaster implements Runnable { - private final Random random = new Random(); + for (Peer peer : getImmutableHandshakedPeers()) { + Message message = peerMessageBuilder.apply(peer); - private List targetPeers; - private Function peerMessageBuilder; - - Broadcaster(List targetPeers, Function peerMessageBuilder) { - this.targetPeers = targetPeers; - this.peerMessageBuilder = peerMessageBuilder; + if (message == null) { + continue; } - @Override - public void run() { - Thread.currentThread().setName("Network Broadcast"); - - for (Peer peer : targetPeers) { - // Very short sleep to reduce strain, improve multi-threading and catch interrupts - try { - Thread.sleep(random.nextInt(20) + 20L); - } catch (InterruptedException e) { - break; - } - - Message message = peerMessageBuilder.apply(peer); - - if (message == null) { - continue; - } - - if (!peer.sendMessage(message)) { - peer.disconnect("failed to broadcast message"); - } - } - - Thread.currentThread().setName("Network Broadcast (dormant)"); + if (!peer.sendMessage(message)) { + peer.disconnect("failed to broadcast message"); } } - - try { - broadcastExecutor.execute(new Broadcaster(this.getImmutableHandshakedPeers(), peerMessageBuilder)); - } catch (RejectedExecutionException e) { - // Can't execute - probably because we're shutting down, so ignore - } } // Shutdown @@ -1521,16 +1487,6 @@ public class Network { LOGGER.warn("Interrupted while waiting for networking threads to terminate"); } - // Stop broadcasts - this.broadcastExecutor.shutdownNow(); - try { - if (!this.broadcastExecutor.awaitTermination(1000, TimeUnit.MILLISECONDS)) { - LOGGER.warn("Broadcast threads failed to terminate"); - } - } catch (InterruptedException e) { - LOGGER.warn("Interrupted while waiting for broadcast threads failed to terminate"); - } - // Close all peer connections for (Peer peer : this.getImmutableConnectedPeers()) { peer.shutdown(); diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index 9d29fc1f..80b888d2 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -664,6 +664,9 @@ public class Peer { LOGGER.trace("[{}] Queuing {} message with ID {} to peer {}", this.peerConnectionId, message.getType().name(), message.getId(), this); + // Check message properly constructed + message.checkValidOutgoing(); + // Possible race condition: // We set OP_WRITE, EPC creates ChannelWriteTask which calls Peer.writeChannel, writeChannel's poll() finds no message to send // Avoided by poll-with-timeout in writeChannel() above. @@ -672,6 +675,9 @@ public class Peer { } catch (InterruptedException e) { // Send failure return false; + } catch (MessageException e) { + LOGGER.error(e.getMessage(), e); + return false; } } diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java index 952010af..ed3cae76 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java @@ -15,27 +15,53 @@ import java.util.List; public class ArbitraryDataFileListMessage extends Message { - private final byte[] signature; - private final List hashes; + private byte[] signature; + private List hashes; private Long requestTime; private Integer requestHops; private String peerAddress; private Boolean isRelayPossible; - public ArbitraryDataFileListMessage(byte[] signature, List hashes, Long requestTime, - Integer requestHops, String peerAddress, boolean isRelayPossible) { + Integer requestHops, String peerAddress, Boolean isRelayPossible) { super(MessageType.ARBITRARY_DATA_FILE_LIST); - this.signature = signature; - this.hashes = hashes; - this.requestTime = requestTime; - this.requestHops = requestHops; - this.peerAddress = peerAddress; - this.isRelayPossible = isRelayPossible; + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(signature); + + bytes.write(Ints.toByteArray(hashes.size())); + + for (byte[] hash : hashes) { + bytes.write(hash); + } + + if (requestTime != null) { + // The remaining fields are optional + + bytes.write(Longs.toByteArray(requestTime)); + + bytes.write(Ints.toByteArray(requestHops)); + + Serialization.serializeSizedStringV2(bytes, peerAddress); + + bytes.write(Ints.toByteArray(Boolean.TRUE.equals(isRelayPossible) ? 1 : 0)); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } - public ArbitraryDataFileListMessage(int id, byte[] signature, List hashes, Long requestTime, + /** Legacy version */ + public ArbitraryDataFileListMessage(byte[] signature, List hashes) { + this(signature, hashes, null, null, null, null); + } + + private ArbitraryDataFileListMessage(int id, byte[] signature, List hashes, Long requestTime, Integer requestHops, String peerAddress, boolean isRelayPossible) { super(id, MessageType.ARBITRARY_DATA_FILE_LIST); @@ -47,12 +73,28 @@ public class ArbitraryDataFileListMessage extends Message { this.isRelayPossible = isRelayPossible; } + public byte[] getSignature() { + return this.signature; + } + public List getHashes() { return this.hashes; } - public byte[] getSignature() { - return this.signature; + public Long getRequestTime() { + return this.requestTime; + } + + public Integer getRequestHops() { + return this.requestHops; + } + + public String getPeerAddress() { + return this.peerAddress; + } + + public Boolean isRelayPossible() { + return this.isRelayPossible; } public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { @@ -74,7 +116,6 @@ public class ArbitraryDataFileListMessage extends Message { boolean isRelayPossible = true; // Legacy versions only send this message when relaying is possible // The remaining fields are optional - if (bytes.hasRemaining()) { try { requestTime = bytes.getLong(); @@ -92,79 +133,4 @@ public class ArbitraryDataFileListMessage extends Message { return new ArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops, peerAddress, isRelayPossible); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - bytes.write(Ints.toByteArray(this.hashes.size())); - - for (byte[] hash : this.hashes) { - bytes.write(hash); - } - - if (this.requestTime == null) { // To maintain backwards support - return bytes.toByteArray(); - } - - // The remaining fields are optional - - bytes.write(Longs.toByteArray(this.requestTime)); - - bytes.write(Ints.toByteArray(this.requestHops)); - - Serialization.serializeSizedStringV2(bytes, this.peerAddress); - - bytes.write(Ints.toByteArray(this.isRelayPossible ? 1 : 0)); - - return bytes.toByteArray(); - } - - public ArbitraryDataFileListMessage cloneWithNewId(int newId) { - ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes, - this.requestTime, this.requestHops, this.peerAddress, this.isRelayPossible); - clone.setId(newId); - return clone; - } - - public void removeOptionalStats() { - this.requestTime = null; - this.requestHops = null; - this.peerAddress = null; - this.isRelayPossible = null; - } - - public Long getRequestTime() { - return this.requestTime; - } - - public void setRequestTime(Long requestTime) { - this.requestTime = requestTime; - } - - public Integer getRequestHops() { - return this.requestHops; - } - - public void setRequestHops(Integer requestHops) { - this.requestHops = requestHops; - } - - public String getPeerAddress() { - return this.peerAddress; - } - - public void setPeerAddress(String peerAddress) { - this.peerAddress = peerAddress; - } - - public Boolean isRelayPossible() { - return this.isRelayPossible; - } - - public void setIsRelayPossible(Boolean isRelayPossible) { - this.isRelayPossible = isRelayPossible; - } - } diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java index 62c5e4d4..50991be3 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java @@ -16,21 +16,39 @@ public class ArbitraryDataFileMessage extends Message { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileMessage.class); - private final byte[] signature; - private final ArbitraryDataFile arbitraryDataFile; + private byte[] signature; + private ArbitraryDataFile arbitraryDataFile; public ArbitraryDataFileMessage(byte[] signature, ArbitraryDataFile arbitraryDataFile) { super(MessageType.ARBITRARY_DATA_FILE); + byte[] data = arbitraryDataFile.getBytes(); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(signature); + + bytes.write(Ints.toByteArray(data.length)); + + bytes.write(data); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private ArbitraryDataFileMessage(int id, byte[] signature, ArbitraryDataFile arbitraryDataFile) { + super(id, MessageType.ARBITRARY_DATA_FILE); + this.signature = signature; this.arbitraryDataFile = arbitraryDataFile; } - public ArbitraryDataFileMessage(int id, byte[] signature, ArbitraryDataFile arbitraryDataFile) { - super(id, MessageType.ARBITRARY_DATA_FILE); - - this.signature = signature; - this.arbitraryDataFile = arbitraryDataFile; + public byte[] getSignature() { + return this.signature; } public ArbitraryDataFile getArbitraryDataFile() { @@ -58,32 +76,4 @@ public class ArbitraryDataFileMessage extends Message { } } - @Override - protected byte[] toData() throws IOException { - if (this.arbitraryDataFile == null) { - return null; - } - - byte[] data = this.arbitraryDataFile.getBytes(); - if (data == null) { - return null; - } - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(signature); - - bytes.write(Ints.toByteArray(data.length)); - - bytes.write(data); - - return bytes.toByteArray(); - } - - public ArbitraryDataFileMessage cloneWithNewId(int newId) { - ArbitraryDataFileMessage clone = new ArbitraryDataFileMessage(this.signature, this.arbitraryDataFile); - clone.setId(newId); - return clone; - } - } diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataMessage.java index b0c42942..142e35cc 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataMessage.java @@ -11,11 +11,26 @@ import com.google.common.primitives.Ints; public class ArbitraryDataMessage extends Message { - private final byte[] signature; - private final byte[] data; + private byte[] signature; + private byte[] data; public ArbitraryDataMessage(byte[] signature, byte[] data) { - this(-1, signature, data); + super(MessageType.ARBITRARY_DATA); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(signature); + + bytes.write(Ints.toByteArray(data.length)); + + bytes.write(data); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private ArbitraryDataMessage(int id, byte[] signature, byte[] data) { @@ -48,20 +63,4 @@ public class ArbitraryDataMessage extends Message { return new ArbitraryDataMessage(id, signature, data); } - @Override - protected byte[] toData() throws IOException { - if (this.data == null) - return null; - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - bytes.write(Ints.toByteArray(this.data.length)); - - bytes.write(this.data); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java b/src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java index fb9a3790..26601d4b 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java @@ -12,21 +12,35 @@ import java.nio.ByteBuffer; public class ArbitraryMetadataMessage extends Message { - private final byte[] signature; - private final ArbitraryDataFile arbitraryMetadataFile; + private byte[] signature; + private ArbitraryDataFile arbitraryMetadataFile; - public ArbitraryMetadataMessage(byte[] signature, ArbitraryDataFile arbitraryDataFile) { + public ArbitraryMetadataMessage(byte[] signature, ArbitraryDataFile arbitraryMetadataFile) { super(MessageType.ARBITRARY_METADATA); - this.signature = signature; - this.arbitraryMetadataFile = arbitraryDataFile; + byte[] data = arbitraryMetadataFile.getBytes(); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(signature); + + bytes.write(Ints.toByteArray(data.length)); + + bytes.write(data); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } - public ArbitraryMetadataMessage(int id, byte[] signature, ArbitraryDataFile arbitraryDataFile) { + private ArbitraryMetadataMessage(int id, byte[] signature, ArbitraryDataFile arbitraryMetadataFile) { super(id, MessageType.ARBITRARY_METADATA); this.signature = signature; - this.arbitraryMetadataFile = arbitraryDataFile; + this.arbitraryMetadataFile = arbitraryMetadataFile; } public byte[] getSignature() { @@ -57,32 +71,4 @@ public class ArbitraryMetadataMessage extends Message { } } - @Override - protected byte[] toData() throws IOException { - if (this.arbitraryMetadataFile == null) { - return null; - } - - byte[] data = this.arbitraryMetadataFile.getBytes(); - if (data == null) { - return null; - } - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(signature); - - bytes.write(Ints.toByteArray(data.length)); - - bytes.write(data); - - return bytes.toByteArray(); - } - - public ArbitraryMetadataMessage cloneWithNewId(int newId) { - ArbitraryMetadataMessage clone = new ArbitraryMetadataMessage(this.signature, this.arbitraryMetadataFile); - clone.setId(newId); - return clone; - } - } diff --git a/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java b/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java index 93c7d5e8..aa75b2a1 100644 --- a/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java @@ -20,7 +20,25 @@ public class ArbitrarySignaturesMessage extends Message { private List signatures; public ArbitrarySignaturesMessage(String peerAddress, int requestHops, List signatures) { - this(-1, peerAddress, requestHops, signatures); + super(MessageType.ARBITRARY_SIGNATURES); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + Serialization.serializeSizedStringV2(bytes, peerAddress); + + bytes.write(Ints.toByteArray(requestHops)); + + bytes.write(Ints.toByteArray(signatures.size())); + + for (byte[] signature : signatures) + bytes.write(signature); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private ArbitrarySignaturesMessage(int id, String peerAddress, int requestHops, List signatures) { @@ -39,14 +57,6 @@ public class ArbitrarySignaturesMessage extends Message { return this.signatures; } - public int getRequestHops() { - return this.requestHops; - } - - public void setRequestHops(int requestHops) { - this.requestHops = requestHops; - } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { String peerAddress; try { @@ -72,20 +82,4 @@ public class ArbitrarySignaturesMessage extends Message { return new ArbitrarySignaturesMessage(id, peerAddress, requestHops, signatures); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - Serialization.serializeSizedStringV2(bytes, this.peerAddress); - - bytes.write(Ints.toByteArray(this.requestHops)); - - bytes.write(Ints.toByteArray(this.signatures.size())); - - for (byte[] signature : this.signatures) - bytes.write(signature); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/BlockMessage.java b/src/main/java/org/qortal/network/message/BlockMessage.java index 8e66257f..2dd4db87 100644 --- a/src/main/java/org/qortal/network/message/BlockMessage.java +++ b/src/main/java/org/qortal/network/message/BlockMessage.java @@ -1,13 +1,10 @@ package org.qortal.network.message; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.block.Block; import org.qortal.data.at.ATStateData; import org.qortal.data.block.BlockData; import org.qortal.data.transaction.TransactionData; @@ -15,27 +12,15 @@ import org.qortal.transform.TransformationException; import org.qortal.transform.block.BlockTransformer; import org.qortal.utils.Triple; -import com.google.common.primitives.Ints; - public class BlockMessage extends Message { private static final Logger LOGGER = LogManager.getLogger(BlockMessage.class); - private Block block = null; + private final BlockData blockData; + private final List transactions; + private final List atStates; - private BlockData blockData = null; - private List transactions = null; - private List atStates = null; - - private final int height; - - public BlockMessage(Block block) { - super(MessageType.BLOCK); - - this.block = block; - this.blockData = block.getBlockData(); - this.height = block.getBlockData().getHeight(); - } + // No public constructor as we're an incoming-only message type. private BlockMessage(int id, BlockData blockData, List transactions, List atStates) { super(id, MessageType.BLOCK); @@ -43,8 +28,6 @@ public class BlockMessage extends Message { this.blockData = blockData; this.transactions = transactions; this.atStates = atStates; - - this.height = blockData.getHeight(); } public BlockData getBlockData() { @@ -75,24 +58,4 @@ public class BlockMessage extends Message { } } - @Override - protected byte[] toData() throws IOException, TransformationException { - if (this.block == null) - return null; - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.height)); - - bytes.write(BlockTransformer.toBytes(this.block)); - - return bytes.toByteArray(); - } - - public BlockMessage cloneWithNewId(int newId) { - BlockMessage clone = new BlockMessage(this.block); - clone.setId(newId); - return clone; - } - } diff --git a/src/main/java/org/qortal/network/message/BlockSummariesMessage.java b/src/main/java/org/qortal/network/message/BlockSummariesMessage.java index 56cc24e3..513e30ae 100644 --- a/src/main/java/org/qortal/network/message/BlockSummariesMessage.java +++ b/src/main/java/org/qortal/network/message/BlockSummariesMessage.java @@ -17,10 +17,28 @@ public class BlockSummariesMessage extends Message { private static final int BLOCK_SUMMARY_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH + Transformer.INT_LENGTH + Transformer.PUBLIC_KEY_LENGTH + Transformer.INT_LENGTH; - private final List blockSummaries; + private List blockSummaries; public BlockSummariesMessage(List blockSummaries) { - this(-1, blockSummaries); + super(MessageType.BLOCK_SUMMARIES); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(blockSummaries.size())); + + for (BlockSummaryData blockSummary : blockSummaries) { + bytes.write(Ints.toByteArray(blockSummary.getHeight())); + bytes.write(blockSummary.getSignature()); + bytes.write(blockSummary.getMinterPublicKey()); + bytes.write(Ints.toByteArray(blockSummary.getOnlineAccountsCount())); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private BlockSummariesMessage(int id, List blockSummaries) { @@ -58,20 +76,4 @@ public class BlockSummariesMessage extends Message { return new BlockSummariesMessage(id, blockSummaries); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.blockSummaries.size())); - - for (BlockSummaryData blockSummary : this.blockSummaries) { - bytes.write(Ints.toByteArray(blockSummary.getHeight())); - bytes.write(blockSummary.getSignature()); - bytes.write(blockSummary.getMinterPublicKey()); - bytes.write(Ints.toByteArray(blockSummary.getOnlineAccountsCount())); - } - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/CachedBlockMessage.java b/src/main/java/org/qortal/network/message/CachedBlockMessage.java index 1a6d79d9..48e9ef36 100644 --- a/src/main/java/org/qortal/network/message/CachedBlockMessage.java +++ b/src/main/java/org/qortal/network/message/CachedBlockMessage.java @@ -11,55 +11,34 @@ import org.qortal.transform.block.BlockTransformer; import com.google.common.primitives.Ints; // This is an OUTGOING-only Message which more readily lends itself to being cached -public class CachedBlockMessage extends Message { +public class CachedBlockMessage extends Message implements Cloneable { - private Block block; - private byte[] cachedBytes = null; - - public CachedBlockMessage(Block block) { + public CachedBlockMessage(Block block) throws TransformationException { super(MessageType.BLOCK); - this.block = block; + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); + + bytes.write(BlockTransformer.toBytes(block)); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } public CachedBlockMessage(byte[] cachedBytes) { super(MessageType.BLOCK); - this.block = null; - this.cachedBytes = cachedBytes; + this.dataBytes = cachedBytes; + this.checksumBytes = Message.generateChecksum(this.dataBytes); } - + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) { throw new UnsupportedOperationException("CachedBlockMessage is for outgoing messages only"); } - @Override - protected byte[] toData() throws IOException, TransformationException { - // Already serialized? - if (this.cachedBytes != null) - return cachedBytes; - - if (this.block == null) - return null; - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.block.getBlockData().getHeight())); - - bytes.write(BlockTransformer.toBytes(this.block)); - - this.cachedBytes = bytes.toByteArray(); - // We no longer need source Block - // and Block contains repository handle which is highly likely to be invalid after this call - this.block = null; - - return this.cachedBytes; - } - - public CachedBlockMessage cloneWithNewId(int newId) { - CachedBlockMessage clone = new CachedBlockMessage(this.cachedBytes); - clone.setId(newId); - return clone; - } - } diff --git a/src/main/java/org/qortal/network/message/ChallengeMessage.java b/src/main/java/org/qortal/network/message/ChallengeMessage.java index 425f9790..bb5b2ae9 100644 --- a/src/main/java/org/qortal/network/message/ChallengeMessage.java +++ b/src/main/java/org/qortal/network/message/ChallengeMessage.java @@ -10,8 +10,25 @@ public class ChallengeMessage extends Message { public static final int CHALLENGE_LENGTH = 32; - private final byte[] publicKey; - private final byte[] challenge; + private byte[] publicKey; + private byte[] challenge; + + public ChallengeMessage(byte[] publicKey, byte[] challenge) { + super(MessageType.CHALLENGE); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(publicKey.length + challenge.length); + + try { + bytes.write(publicKey); + + bytes.write(challenge); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } private ChallengeMessage(int id, byte[] publicKey, byte[] challenge) { super(id, MessageType.CHALLENGE); @@ -20,10 +37,6 @@ public class ChallengeMessage extends Message { this.challenge = challenge; } - public ChallengeMessage(byte[] publicKey, byte[] challenge) { - this(-1, publicKey, challenge); - } - public byte[] getPublicKey() { return this.publicKey; } @@ -42,15 +55,4 @@ public class ChallengeMessage extends Message { return new ChallengeMessage(id, publicKey, challenge); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.publicKey); - - bytes.write(this.challenge); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java index 69668242..467a229f 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java @@ -15,14 +15,44 @@ import java.util.List; public class GetArbitraryDataFileListMessage extends Message { - private final byte[] signature; + private byte[] signature; private List hashes; - private final long requestTime; + private long requestTime; private int requestHops; private String requestingPeer; public GetArbitraryDataFileListMessage(byte[] signature, List hashes, long requestTime, int requestHops, String requestingPeer) { - this(-1, signature, hashes, requestTime, requestHops, requestingPeer); + super(MessageType.GET_ARBITRARY_DATA_FILE_LIST); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(signature); + + bytes.write(Longs.toByteArray(requestTime)); + + bytes.write(Ints.toByteArray(requestHops)); + + if (hashes != null) { + bytes.write(Ints.toByteArray(hashes.size())); + + for (byte[] hash : hashes) { + bytes.write(hash); + } + } + else { + bytes.write(Ints.toByteArray(0)); + } + + if (requestingPeer != null) { + Serialization.serializeSizedStringV2(bytes, requestingPeer); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetArbitraryDataFileListMessage(int id, byte[] signature, List hashes, long requestTime, int requestHops, String requestingPeer) { @@ -43,6 +73,18 @@ public class GetArbitraryDataFileListMessage extends Message { return this.hashes; } + public long getRequestTime() { + return this.requestTime; + } + + public int getRequestHops() { + return this.requestHops; + } + + public String getRequestingPeer() { + return this.requestingPeer; + } + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; @@ -76,47 +118,4 @@ public class GetArbitraryDataFileListMessage extends Message { return new GetArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops, requestingPeer); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - bytes.write(Longs.toByteArray(this.requestTime)); - - bytes.write(Ints.toByteArray(this.requestHops)); - - if (this.hashes != null) { - bytes.write(Ints.toByteArray(this.hashes.size())); - - for (byte[] hash : this.hashes) { - bytes.write(hash); - } - } - else { - bytes.write(Ints.toByteArray(0)); - } - - if (this.requestingPeer != null) { - Serialization.serializeSizedStringV2(bytes, this.requestingPeer); - } - - return bytes.toByteArray(); - } - - public long getRequestTime() { - return this.requestTime; - } - - public int getRequestHops() { - return this.requestHops; - } - public void setRequestHops(int requestHops) { - this.requestHops = requestHops; - } - - public String getRequestingPeer() { - return this.requestingPeer; - } - } diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java index 5ec5aa82..d97a4847 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java @@ -8,11 +8,24 @@ import java.nio.ByteBuffer; public class GetArbitraryDataFileMessage extends Message { - private final byte[] signature; - private final byte[] hash; + private byte[] signature; + private byte[] hash; public GetArbitraryDataFileMessage(byte[] signature, byte[] hash) { - this(-1, signature, hash); + super(MessageType.GET_ARBITRARY_DATA_FILE); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(signature.length + hash.length); + + try { + bytes.write(signature); + + bytes.write(hash); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetArbitraryDataFileMessage(int id, byte[] signature, byte[] hash) { @@ -40,15 +53,4 @@ public class GetArbitraryDataFileMessage extends Message { return new GetArbitraryDataFileMessage(id, signature, hash); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - bytes.write(this.hash); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataMessage.java index df437477..bf604fe7 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryDataMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataMessage.java @@ -1,17 +1,19 @@ package org.qortal.network.message; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import org.qortal.transform.Transformer; public class GetArbitraryDataMessage extends Message { - private final byte[] signature; + private byte[] signature; public GetArbitraryDataMessage(byte[] signature) { - this(-1, signature); + super(MessageType.GET_ARBITRARY_DATA); + + this.dataBytes = Arrays.copyOf(signature, signature.length); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetArbitraryDataMessage(int id, byte[] signature) { @@ -32,13 +34,4 @@ public class GetArbitraryDataMessage extends Message { return new GetArbitraryDataMessage(id, signature); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java index a4b43e41..2501d5c3 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java @@ -10,12 +10,27 @@ import java.nio.ByteBuffer; public class GetArbitraryMetadataMessage extends Message { - private final byte[] signature; - private final long requestTime; + private byte[] signature; + private long requestTime; private int requestHops; public GetArbitraryMetadataMessage(byte[] signature, long requestTime, int requestHops) { - this(-1, signature, requestTime, requestHops); + super(MessageType.GET_ARBITRARY_METADATA); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(signature); + + bytes.write(Longs.toByteArray(requestTime)); + + bytes.write(Ints.toByteArray(requestHops)); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetArbitraryMetadataMessage(int id, byte[] signature, long requestTime, int requestHops) { @@ -30,6 +45,14 @@ public class GetArbitraryMetadataMessage extends Message { return this.signature; } + public long getRequestTime() { + return this.requestTime; + } + + public int getRequestHops() { + return this.requestHops; + } + public static Message fromByteBuffer(int id, ByteBuffer bytes) { byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); @@ -41,29 +64,4 @@ public class GetArbitraryMetadataMessage extends Message { return new GetArbitraryMetadataMessage(id, signature, requestTime, requestHops); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - bytes.write(Longs.toByteArray(this.requestTime)); - - bytes.write(Ints.toByteArray(this.requestHops)); - - return bytes.toByteArray(); - } - - public long getRequestTime() { - return this.requestTime; - } - - public int getRequestHops() { - return this.requestHops; - } - - public void setRequestHops(int requestHops) { - this.requestHops = requestHops; - } - } diff --git a/src/main/java/org/qortal/network/message/GetBlockMessage.java b/src/main/java/org/qortal/network/message/GetBlockMessage.java index 538c46cd..d39dcca0 100644 --- a/src/main/java/org/qortal/network/message/GetBlockMessage.java +++ b/src/main/java/org/qortal/network/message/GetBlockMessage.java @@ -1,17 +1,19 @@ package org.qortal.network.message; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import org.qortal.transform.block.BlockTransformer; public class GetBlockMessage extends Message { - private final byte[] signature; + private byte[] signature; public GetBlockMessage(byte[] signature) { - this(-1, signature); + super(MessageType.GET_BLOCK); + + this.dataBytes = Arrays.copyOf(signature, signature.length); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetBlockMessage(int id, byte[] signature) { @@ -31,13 +33,4 @@ public class GetBlockMessage extends Message { return new GetBlockMessage(id, signature); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/GetBlockSummariesMessage.java b/src/main/java/org/qortal/network/message/GetBlockSummariesMessage.java index e398d532..70f0d5c5 100644 --- a/src/main/java/org/qortal/network/message/GetBlockSummariesMessage.java +++ b/src/main/java/org/qortal/network/message/GetBlockSummariesMessage.java @@ -10,11 +10,24 @@ import com.google.common.primitives.Ints; public class GetBlockSummariesMessage extends Message { - private final byte[] parentSignature; - private final int numberRequested; + private byte[] parentSignature; + private int numberRequested; public GetBlockSummariesMessage(byte[] parentSignature, int numberRequested) { - this(-1, parentSignature, numberRequested); + super(MessageType.GET_BLOCK_SUMMARIES); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(parentSignature); + + bytes.write(Ints.toByteArray(numberRequested)); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetBlockSummariesMessage(int id, byte[] parentSignature, int numberRequested) { @@ -41,15 +54,4 @@ public class GetBlockSummariesMessage extends Message { return new GetBlockSummariesMessage(id, parentSignature, numberRequested); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.parentSignature); - - bytes.write(Ints.toByteArray(this.numberRequested)); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java index 9112ba1a..ae98cf40 100644 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java @@ -16,10 +16,27 @@ import com.google.common.primitives.Longs; public class GetOnlineAccountsMessage extends Message { private static final int MAX_ACCOUNT_COUNT = 5000; - private final List onlineAccounts; + private List onlineAccounts; public GetOnlineAccountsMessage(List onlineAccounts) { - this(-1, onlineAccounts); + super(MessageType.GET_ONLINE_ACCOUNTS); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(onlineAccounts.size())); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); + + bytes.write(onlineAccountData.getPublicKey()); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetOnlineAccountsMessage(int id, List onlineAccounts) { @@ -49,19 +66,4 @@ public class GetOnlineAccountsMessage extends Message { return new GetOnlineAccountsMessage(id, onlineAccounts); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.onlineAccounts.size())); - - for (OnlineAccountData onlineAccountData : this.onlineAccounts) { - bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); - - bytes.write(onlineAccountData.getPublicKey()); - } - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java index bc820ff1..d3df3a1b 100644 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java @@ -23,11 +23,43 @@ import java.util.Map; * Also V2 only builds online accounts message once! */ public class GetOnlineAccountsV2Message extends Message { - private final List onlineAccounts; - private byte[] cachedData; + + private List onlineAccounts; public GetOnlineAccountsV2Message(List onlineAccounts) { - this(-1, onlineAccounts); + super(MessageType.GET_ONLINE_ACCOUNTS_V2); + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + Long timestamp = onlineAccountData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH; + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + try { + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + if (onlineAccountData.getTimestamp() == timestamp) + bytes.write(onlineAccountData.getPublicKey()); + } + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetOnlineAccountsV2Message(int id, List onlineAccounts) { @@ -66,44 +98,4 @@ public class GetOnlineAccountsV2Message extends Message { return new GetOnlineAccountsV2Message(id, onlineAccounts); } - @Override - protected synchronized byte[] toData() throws IOException { - if (this.cachedData != null) - return this.cachedData; - - // Shortcut in case we have no online accounts - if (this.onlineAccounts.isEmpty()) { - this.cachedData = Ints.toByteArray(0); - return this.cachedData; - } - - // How many of each timestamp - Map countByTimestamp = new HashMap<>(); - - for (OnlineAccountData onlineAccountData : this.onlineAccounts) { - Long timestamp = onlineAccountData.getTimestamp(); - countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); - } - - // We should know exactly how many bytes to allocate now - int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) - + this.onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH; - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - - bytes.write(Longs.toByteArray(timestamp)); - - for (OnlineAccountData onlineAccountData : this.onlineAccounts) { - if (onlineAccountData.getTimestamp() == timestamp) - bytes.write(onlineAccountData.getPublicKey()); - } - } - - this.cachedData = bytes.toByteArray(); - return this.cachedData; - } - } diff --git a/src/main/java/org/qortal/network/message/GetPeersMessage.java b/src/main/java/org/qortal/network/message/GetPeersMessage.java index 4918b92d..b8f7e128 100644 --- a/src/main/java/org/qortal/network/message/GetPeersMessage.java +++ b/src/main/java/org/qortal/network/message/GetPeersMessage.java @@ -5,7 +5,9 @@ import java.nio.ByteBuffer; public class GetPeersMessage extends Message { public GetPeersMessage() { - this(-1); + super(MessageType.GET_PEERS); + + this.dataBytes = EMPTY_DATA_BYTES; } private GetPeersMessage(int id) { @@ -16,9 +18,4 @@ public class GetPeersMessage extends Message { return new GetPeersMessage(id); } - @Override - protected byte[] toData() { - return new byte[0]; - } - } diff --git a/src/main/java/org/qortal/network/message/GetSignaturesV2Message.java b/src/main/java/org/qortal/network/message/GetSignaturesV2Message.java index f0ac2265..0f88ba7d 100644 --- a/src/main/java/org/qortal/network/message/GetSignaturesV2Message.java +++ b/src/main/java/org/qortal/network/message/GetSignaturesV2Message.java @@ -10,11 +10,24 @@ import com.google.common.primitives.Ints; public class GetSignaturesV2Message extends Message { - private final byte[] parentSignature; - private final int numberRequested; + private byte[] parentSignature; + private int numberRequested; public GetSignaturesV2Message(byte[] parentSignature, int numberRequested) { - this(-1, parentSignature, numberRequested); + super(MessageType.GET_SIGNATURES_V2); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(parentSignature); + + bytes.write(Ints.toByteArray(numberRequested)); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetSignaturesV2Message(int id, byte[] parentSignature, int numberRequested) { @@ -41,15 +54,4 @@ public class GetSignaturesV2Message extends Message { return new GetSignaturesV2Message(id, parentSignature, numberRequested); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.parentSignature); - - bytes.write(Ints.toByteArray(this.numberRequested)); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/GetTradePresencesMessage.java b/src/main/java/org/qortal/network/message/GetTradePresencesMessage.java index 27fd2f9d..7246c424 100644 --- a/src/main/java/org/qortal/network/message/GetTradePresencesMessage.java +++ b/src/main/java/org/qortal/network/message/GetTradePresencesMessage.java @@ -19,11 +19,49 @@ import java.util.Map; * Groups of: number of entries, timestamp, then AT trade pubkey for each entry. */ public class GetTradePresencesMessage extends Message { - private final List tradePresences; - private byte[] cachedData; + private List tradePresences; public GetTradePresencesMessage(List tradePresences) { - this(-1, tradePresences); + super(MessageType.GET_TRADE_PRESENCES); + + // Shortcut in case we have no trade presences + if (tradePresences.isEmpty()) { + this.dataBytes = Ints.toByteArray(0); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + return; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (TradePresenceData tradePresenceData : tradePresences) { + Long timestamp = tradePresenceData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + tradePresences.size() * Transformer.PUBLIC_KEY_LENGTH; + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + try { + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (TradePresenceData tradePresenceData : tradePresences) { + if (tradePresenceData.getTimestamp() == timestamp) + bytes.write(tradePresenceData.getPublicKey()); + } + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetTradePresencesMessage(int id, List tradePresences) { @@ -62,44 +100,4 @@ public class GetTradePresencesMessage extends Message { return new GetTradePresencesMessage(id, tradePresences); } - @Override - protected synchronized byte[] toData() throws IOException { - if (this.cachedData != null) - return this.cachedData; - - // Shortcut in case we have no trade presences - if (this.tradePresences.isEmpty()) { - this.cachedData = Ints.toByteArray(0); - return this.cachedData; - } - - // How many of each timestamp - Map countByTimestamp = new HashMap<>(); - - for (TradePresenceData tradePresenceData : this.tradePresences) { - Long timestamp = tradePresenceData.getTimestamp(); - countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); - } - - // We should know exactly how many bytes to allocate now - int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) - + this.tradePresences.size() * Transformer.PUBLIC_KEY_LENGTH; - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - - bytes.write(Longs.toByteArray(timestamp)); - - for (TradePresenceData tradePresenceData : this.tradePresences) { - if (tradePresenceData.getTimestamp() == timestamp) - bytes.write(tradePresenceData.getPublicKey()); - } - } - - this.cachedData = bytes.toByteArray(); - return this.cachedData; - } - } diff --git a/src/main/java/org/qortal/network/message/GetTransactionMessage.java b/src/main/java/org/qortal/network/message/GetTransactionMessage.java index a0153aed..fe0c750f 100644 --- a/src/main/java/org/qortal/network/message/GetTransactionMessage.java +++ b/src/main/java/org/qortal/network/message/GetTransactionMessage.java @@ -1,17 +1,19 @@ package org.qortal.network.message; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import org.qortal.transform.Transformer; public class GetTransactionMessage extends Message { - private final byte[] signature; + private byte[] signature; public GetTransactionMessage(byte[] signature) { - this(-1, signature); + super(MessageType.GET_TRANSACTION); + + this.dataBytes = Arrays.copyOf(signature, signature.length); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetTransactionMessage(int id, byte[] signature) { @@ -32,13 +34,4 @@ public class GetTransactionMessage extends Message { return new GetTransactionMessage(id, signature); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/GetUnconfirmedTransactionsMessage.java b/src/main/java/org/qortal/network/message/GetUnconfirmedTransactionsMessage.java index fc018e6f..fccd4c74 100644 --- a/src/main/java/org/qortal/network/message/GetUnconfirmedTransactionsMessage.java +++ b/src/main/java/org/qortal/network/message/GetUnconfirmedTransactionsMessage.java @@ -5,7 +5,9 @@ import java.nio.ByteBuffer; public class GetUnconfirmedTransactionsMessage extends Message { public GetUnconfirmedTransactionsMessage() { - this(-1); + super(MessageType.GET_UNCONFIRMED_TRANSACTIONS); + + this.dataBytes = EMPTY_DATA_BYTES; } private GetUnconfirmedTransactionsMessage(int id) { @@ -16,9 +18,4 @@ public class GetUnconfirmedTransactionsMessage extends Message { return new GetUnconfirmedTransactionsMessage(id); } - @Override - protected byte[] toData() { - return new byte[0]; - } - } diff --git a/src/main/java/org/qortal/network/message/GoodbyeMessage.java b/src/main/java/org/qortal/network/message/GoodbyeMessage.java index 8c0c221e..74130be2 100644 --- a/src/main/java/org/qortal/network/message/GoodbyeMessage.java +++ b/src/main/java/org/qortal/network/message/GoodbyeMessage.java @@ -3,7 +3,6 @@ package org.qortal.network.message; import static java.util.Arrays.stream; import static java.util.stream.Collectors.toMap; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.Map; @@ -31,7 +30,14 @@ public class GoodbyeMessage extends Message { } } - private final Reason reason; + private Reason reason; + + public GoodbyeMessage(Reason reason) { + super(MessageType.GOODBYE); + + this.dataBytes = Ints.toByteArray(reason.value); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } private GoodbyeMessage(int id, Reason reason) { super(id, MessageType.GOODBYE); @@ -39,10 +45,6 @@ public class GoodbyeMessage extends Message { this.reason = reason; } - public GoodbyeMessage(Reason reason) { - this(-1, reason); - } - public Reason getReason() { return this.reason; } @@ -57,9 +59,4 @@ public class GoodbyeMessage extends Message { return new GoodbyeMessage(id, reason); } - @Override - protected byte[] toData() throws IOException { - return Ints.toByteArray(this.reason.value); - } - } diff --git a/src/main/java/org/qortal/network/message/HeightV2Message.java b/src/main/java/org/qortal/network/message/HeightV2Message.java index 3d6a310c..0e775a84 100644 --- a/src/main/java/org/qortal/network/message/HeightV2Message.java +++ b/src/main/java/org/qortal/network/message/HeightV2Message.java @@ -12,13 +12,30 @@ import com.google.common.primitives.Longs; public class HeightV2Message extends Message { - private final int height; - private final byte[] signature; - private final long timestamp; - private final byte[] minterPublicKey; + private int height; + private byte[] signature; + private long timestamp; + private byte[] minterPublicKey; public HeightV2Message(int height, byte[] signature, long timestamp, byte[] minterPublicKey) { - this(-1, height, signature, timestamp, minterPublicKey); + super(MessageType.HEIGHT_V2); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(height)); + + bytes.write(signature); + + bytes.write(Longs.toByteArray(timestamp)); + + bytes.write(minterPublicKey); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private HeightV2Message(int id, int height, byte[] signature, long timestamp, byte[] minterPublicKey) { @@ -60,19 +77,4 @@ public class HeightV2Message extends Message { return new HeightV2Message(id, height, signature, timestamp, minterPublicKey); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.height)); - - bytes.write(this.signature); - - bytes.write(Longs.toByteArray(this.timestamp)); - - bytes.write(this.minterPublicKey); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/HelloMessage.java b/src/main/java/org/qortal/network/message/HelloMessage.java index 80314c2e..30b7d9be 100644 --- a/src/main/java/org/qortal/network/message/HelloMessage.java +++ b/src/main/java/org/qortal/network/message/HelloMessage.java @@ -11,9 +11,28 @@ import com.google.common.primitives.Longs; public class HelloMessage extends Message { - private final long timestamp; - private final String versionString; - private final String senderPeerAddress; + private long timestamp; + private String versionString; + private String senderPeerAddress; + + public HelloMessage(long timestamp, String versionString, String senderPeerAddress) { + super(MessageType.HELLO); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Longs.toByteArray(timestamp)); + + Serialization.serializeSizedString(bytes, versionString); + + Serialization.serializeSizedString(bytes, senderPeerAddress); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } private HelloMessage(int id, long timestamp, String versionString, String senderPeerAddress) { super(id, MessageType.HELLO); @@ -23,10 +42,6 @@ public class HelloMessage extends Message { this.senderPeerAddress = senderPeerAddress; } - public HelloMessage(long timestamp, String versionString, String senderPeerAddress) { - this(-1, timestamp, versionString, senderPeerAddress); - } - public long getTimestamp() { return this.timestamp; } @@ -58,17 +73,4 @@ public class HelloMessage extends Message { return new HelloMessage(id, timestamp, versionString, senderPeerAddress); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Longs.toByteArray(this.timestamp)); - - Serialization.serializeSizedString(bytes, this.versionString); - - Serialization.serializeSizedString(bytes, this.senderPeerAddress); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/Message.java b/src/main/java/org/qortal/network/message/Message.java index 15e8cb4f..e92aca89 100644 --- a/src/main/java/org/qortal/network/message/Message.java +++ b/src/main/java/org/qortal/network/message/Message.java @@ -2,35 +2,66 @@ package org.qortal.network.message; import org.qortal.crypto.Crypto; import org.qortal.network.Network; -import org.qortal.transform.TransformationException; import com.google.common.primitives.Ints; -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.toMap; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.Arrays; +/** + * Network message for sending over network, or unpacked data received from network. + *

+ *

+ * For messages received from network, subclass's {@code fromByteBuffer()} method is used + * to construct a subclassed instance. Original bytes from network are not retained. + * Access to deserialized data should be via subclass's getters. Ideally there should be NO setters! + *

+ *

+ *

+ * Each subclass's public constructor is for building a message to send only. + * The constructor will serialize into byte form but not store the passed args. + * Serialized bytes are saved into superclass (Message) {@code dataBytes} and, if not empty, + * a checksum is created and saved into {@code checksumBytes}. + * Therefore: do not use subclass's getters after using constructor! + *

+ *

+ *

+ * For subclasses where outgoing versions might be usefully cached, they can implement Clonable + * as long if they are safe to use {@link Object#clone()}. + *

+ */ public abstract class Message { // MAGIC(4) + TYPE(4) + HAS-ID(1) + ID?(4) + DATA-SIZE(4) + CHECKSUM?(4) + DATA?(*) private static final int MAGIC_LENGTH = 4; + private static final int TYPE_LENGTH = 4; + private static final int HAS_ID_LENGTH = 1; + private static final int ID_LENGTH = 4; + private static final int DATA_SIZE_LENGTH = 4; private static final int CHECKSUM_LENGTH = 4; private static final int MAX_DATA_SIZE = 10 * 1024 * 1024; // 10MB - private int id; - private MessageType type; + protected static final byte[] EMPTY_DATA_BYTES = new byte[0]; + protected int id; + protected final MessageType type; + + /** Serialized outgoing message data. Expected to be written to by subclass. */ + protected byte[] dataBytes; + /** Serialized outgoing message checksum. Expected to be written to by subclass. */ + protected byte[] checksumBytes; + + /** Typically called by subclass when constructing message from received network data. */ protected Message(int id, MessageType type) { this.id = id; this.type = type; } + /** Typically called by subclass when constructing outgoing message. */ protected Message(MessageType type) { this(-1, type); } @@ -54,7 +85,7 @@ public abstract class Message { /** * Attempt to read a message from byte buffer. * - * @param readOnlyBuffer + * @param readOnlyBuffer ByteBuffer containing bytes read from network * @return null if no complete message can be read * @throws MessageException if message could not be decoded or is invalid */ @@ -131,9 +162,27 @@ public abstract class Message { return Arrays.copyOfRange(Crypto.digest(dataBuffer), 0, CHECKSUM_LENGTH); } + public void checkValidOutgoing() throws MessageException { + // We expect subclass to have initialized these + if (this.dataBytes == null) + throw new MessageException("Missing data payload"); + if (this.dataBytes.length > 0 && this.checksumBytes == null) + throw new MessageException("Missing data checksum"); + } + public byte[] toBytes() throws MessageException { + checkValidOutgoing(); + + // We can calculate exact length + int messageLength = MAGIC_LENGTH + TYPE_LENGTH + HAS_ID_LENGTH; + messageLength += this.hasId() ? ID_LENGTH : 0; + messageLength += DATA_SIZE_LENGTH + this.dataBytes.length > 0 ? CHECKSUM_LENGTH + this.dataBytes.length : 0; + + if (messageLength > MAX_DATA_SIZE) + throw new MessageException(String.format("About to send message with length %d larger than allowed %d", messageLength, MAX_DATA_SIZE)); + try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(256); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(messageLength); // Magic bytes.write(Network.getInstance().getMessageMagic()); @@ -148,32 +197,30 @@ public abstract class Message { bytes.write(0); } - byte[] data = this.toData(); - if (data == null) - throw new MessageException("Missing data payload"); + bytes.write(Ints.toByteArray(this.dataBytes.length)); - bytes.write(Ints.toByteArray(data.length)); - - if (data.length > 0) { - bytes.write(generateChecksum(data)); - bytes.write(data); + if (this.dataBytes.length > 0) { + bytes.write(this.checksumBytes); + bytes.write(this.dataBytes); } - if (bytes.size() > MAX_DATA_SIZE) - throw new MessageException(String.format("About to send message with length %d larger than allowed %d", bytes.size(), MAX_DATA_SIZE)); - return bytes.toByteArray(); - } catch (IOException | TransformationException e) { + } catch (IOException e) { throw new MessageException("Failed to serialize message", e); } } - /** Serialize message into bytes. - * - * @return message as byte array, or null if message is missing payload data / uninitialized somehow - * @throws IOException if unable / failed to serialize - * @throws TransformationException if unable / failed to serialize - */ - protected abstract byte[] toData() throws IOException, TransformationException; + public static M cloneWithNewId(M message, int newId) { + M clone; + + try { + clone = (M) message.clone(); + } catch (CloneNotSupportedException e) { + throw new UnsupportedOperationException("Message sub-class not cloneable"); + } + + clone.setId(newId); + return clone; + } } diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java index d7acb2fd..e7e4c32c 100644 --- a/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java +++ b/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java @@ -16,10 +16,29 @@ import com.google.common.primitives.Longs; public class OnlineAccountsMessage extends Message { private static final int MAX_ACCOUNT_COUNT = 5000; - private final List onlineAccounts; + private List onlineAccounts; public OnlineAccountsMessage(List onlineAccounts) { - this(-1, onlineAccounts); + super(MessageType.ONLINE_ACCOUNTS); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(onlineAccounts.size())); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); + + bytes.write(onlineAccountData.getSignature()); + + bytes.write(onlineAccountData.getPublicKey()); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private OnlineAccountsMessage(int id, List onlineAccounts) { @@ -53,21 +72,4 @@ public class OnlineAccountsMessage extends Message { return new OnlineAccountsMessage(id, onlineAccounts); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.onlineAccounts.size())); - - for (OnlineAccountData onlineAccountData : this.onlineAccounts) { - bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); - - bytes.write(onlineAccountData.getSignature()); - - bytes.write(onlineAccountData.getPublicKey()); - } - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java index 955237fe..6803e3bf 100644 --- a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java +++ b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java @@ -23,11 +23,52 @@ import java.util.Map; * Also V2 only builds online accounts message once! */ public class OnlineAccountsV2Message extends Message { - private final List onlineAccounts; - private byte[] cachedData; + + private List onlineAccounts; public OnlineAccountsV2Message(List onlineAccounts) { - this(-1, onlineAccounts); + super(MessageType.ONLINE_ACCOUNTS_V2); + + // Shortcut in case we have no online accounts + if (onlineAccounts.isEmpty()) { + this.dataBytes = Ints.toByteArray(0); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + return; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + Long timestamp = onlineAccountData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + try { + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + if (onlineAccountData.getTimestamp() == timestamp) { + bytes.write(onlineAccountData.getSignature()); + bytes.write(onlineAccountData.getPublicKey()); + } + } + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private OnlineAccountsV2Message(int id, List onlineAccounts) { @@ -69,46 +110,4 @@ public class OnlineAccountsV2Message extends Message { return new OnlineAccountsV2Message(id, onlineAccounts); } - @Override - protected synchronized byte[] toData() throws IOException { - if (this.cachedData != null) - return this.cachedData; - - // Shortcut in case we have no online accounts - if (this.onlineAccounts.isEmpty()) { - this.cachedData = Ints.toByteArray(0); - return this.cachedData; - } - - // How many of each timestamp - Map countByTimestamp = new HashMap<>(); - - for (OnlineAccountData onlineAccountData : this.onlineAccounts) { - Long timestamp = onlineAccountData.getTimestamp(); - countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); - } - - // We should know exactly how many bytes to allocate now - int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) - + this.onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH); - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - - bytes.write(Longs.toByteArray(timestamp)); - - for (OnlineAccountData onlineAccountData : this.onlineAccounts) { - if (onlineAccountData.getTimestamp() == timestamp) { - bytes.write(onlineAccountData.getSignature()); - bytes.write(onlineAccountData.getPublicKey()); - } - } - } - - this.cachedData = bytes.toByteArray(); - return this.cachedData; - } - } diff --git a/src/main/java/org/qortal/network/message/PeersV2Message.java b/src/main/java/org/qortal/network/message/PeersV2Message.java index 4166c37d..e844246f 100644 --- a/src/main/java/org/qortal/network/message/PeersV2Message.java +++ b/src/main/java/org/qortal/network/message/PeersV2Message.java @@ -15,10 +15,38 @@ import com.google.common.primitives.Ints; // NOTE: this message supports hostnames, literal IP addresses (IPv4 and IPv6) with port numbers public class PeersV2Message extends Message { - private final List peerAddresses; + private List peerAddresses; public PeersV2Message(List peerAddresses) { - this(-1, peerAddresses); + super(MessageType.PEERS_V2); + + List addresses = new ArrayList<>(); + + // First entry represents sending node but contains only port number with empty address. + addresses.add(("0.0.0.0:" + Settings.getInstance().getListenPort()).getBytes(StandardCharsets.UTF_8)); + + for (PeerAddress peerAddress : peerAddresses) + addresses.add(peerAddress.toString().getBytes(StandardCharsets.UTF_8)); + + // We can't send addresses that are longer than 255 bytes as length itself is encoded in one byte. + addresses.removeIf(addressString -> addressString.length > 255); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + // Number of entries + bytes.write(Ints.toByteArray(addresses.size())); + + for (byte[] address : addresses) { + bytes.write(address.length); + bytes.write(address); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private PeersV2Message(int id, List peerAddresses) { @@ -55,32 +83,4 @@ public class PeersV2Message extends Message { return new PeersV2Message(id, peerAddresses); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - List addresses = new ArrayList<>(); - - // First entry represents sending node but contains only port number with empty address. - addresses.add(("0.0.0.0:" + Settings.getInstance().getListenPort()).getBytes(StandardCharsets.UTF_8)); - - for (PeerAddress peerAddress : this.peerAddresses) - addresses.add(peerAddress.toString().getBytes(StandardCharsets.UTF_8)); - - // We can't send addresses that are longer than 255 bytes as length itself is encoded in one byte. - addresses.removeIf(addressString -> addressString.length > 255); - - // Serialize - - // Number of entries - bytes.write(Ints.toByteArray(addresses.size())); - - for (byte[] address : addresses) { - bytes.write(address.length); - bytes.write(address); - } - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/PingMessage.java b/src/main/java/org/qortal/network/message/PingMessage.java index fa04fdb6..0b66d507 100644 --- a/src/main/java/org/qortal/network/message/PingMessage.java +++ b/src/main/java/org/qortal/network/message/PingMessage.java @@ -5,7 +5,9 @@ import java.nio.ByteBuffer; public class PingMessage extends Message { public PingMessage() { - this(-1); + super(MessageType.PING); + + this.dataBytes = EMPTY_DATA_BYTES; } private PingMessage(int id) { @@ -16,9 +18,4 @@ public class PingMessage extends Message { return new PingMessage(id); } - @Override - protected byte[] toData() { - return new byte[0]; - } - } diff --git a/src/main/java/org/qortal/network/message/PongMessage.java b/src/main/java/org/qortal/network/message/PongMessage.java index 4338aeac..4e73c07c 100644 --- a/src/main/java/org/qortal/network/message/PongMessage.java +++ b/src/main/java/org/qortal/network/message/PongMessage.java @@ -5,7 +5,9 @@ import java.nio.ByteBuffer; public class PongMessage extends Message { public PongMessage() { - this(-1); + super(MessageType.PONG); + + this.dataBytes = EMPTY_DATA_BYTES; } private PongMessage(int id) { @@ -16,9 +18,4 @@ public class PongMessage extends Message { return new PongMessage(id); } - @Override - protected byte[] toData() { - return new byte[0]; - } - } diff --git a/src/main/java/org/qortal/network/message/ResponseMessage.java b/src/main/java/org/qortal/network/message/ResponseMessage.java index 6fed6d6a..292fe697 100644 --- a/src/main/java/org/qortal/network/message/ResponseMessage.java +++ b/src/main/java/org/qortal/network/message/ResponseMessage.java @@ -10,8 +10,25 @@ public class ResponseMessage extends Message { public static final int DATA_LENGTH = 32; - private final int nonce; - private final byte[] data; + private int nonce; + private byte[] data; + + public ResponseMessage(int nonce, byte[] data) { + super(MessageType.RESPONSE); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(4 + DATA_LENGTH); + + try { + bytes.write(Ints.toByteArray(nonce)); + + bytes.write(data); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } private ResponseMessage(int id, int nonce, byte[] data) { super(id, MessageType.RESPONSE); @@ -20,10 +37,6 @@ public class ResponseMessage extends Message { this.data = data; } - public ResponseMessage(int nonce, byte[] data) { - this(-1, nonce, data); - } - public int getNonce() { return this.nonce; } @@ -41,15 +54,4 @@ public class ResponseMessage extends Message { return new ResponseMessage(id, nonce, data); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(4 + DATA_LENGTH); - - bytes.write(Ints.toByteArray(this.nonce)); - - bytes.write(data); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/SignaturesMessage.java b/src/main/java/org/qortal/network/message/SignaturesMessage.java index cbbaf790..c0b44fcd 100644 --- a/src/main/java/org/qortal/network/message/SignaturesMessage.java +++ b/src/main/java/org/qortal/network/message/SignaturesMessage.java @@ -13,10 +13,24 @@ import com.google.common.primitives.Ints; public class SignaturesMessage extends Message { - private final List signatures; + private List signatures; public SignaturesMessage(List signatures) { - this(-1, signatures); + super(MessageType.SIGNATURES); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(signatures.size())); + + for (byte[] signature : signatures) + bytes.write(signature); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private SignaturesMessage(int id, List signatures) { @@ -45,16 +59,4 @@ public class SignaturesMessage extends Message { return new SignaturesMessage(id, signatures); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.signatures.size())); - - for (byte[] signature : this.signatures) - bytes.write(signature); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/TradePresencesMessage.java b/src/main/java/org/qortal/network/message/TradePresencesMessage.java index 20edfdaf..8d7da156 100644 --- a/src/main/java/org/qortal/network/message/TradePresencesMessage.java +++ b/src/main/java/org/qortal/network/message/TradePresencesMessage.java @@ -20,11 +20,55 @@ import java.util.Map; * Groups of: number of entries, timestamp, then pubkey + sig + AT address for each entry. */ public class TradePresencesMessage extends Message { - private final List tradePresences; - private byte[] cachedData; + + private List tradePresences; public TradePresencesMessage(List tradePresences) { - this(-1, tradePresences); + super(MessageType.TRADE_PRESENCES); + + // Shortcut in case we have no trade presences + if (tradePresences.isEmpty()) { + this.dataBytes = Ints.toByteArray(0); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + return; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (TradePresenceData tradePresenceData : tradePresences) { + Long timestamp = tradePresenceData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + tradePresences.size() * (Transformer.PUBLIC_KEY_LENGTH + Transformer.SIGNATURE_LENGTH + Transformer.ADDRESS_LENGTH); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + try { + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (TradePresenceData tradePresenceData : tradePresences) { + if (tradePresenceData.getTimestamp() == timestamp) { + bytes.write(tradePresenceData.getPublicKey()); + + bytes.write(tradePresenceData.getSignature()); + + bytes.write(Base58.decode(tradePresenceData.getAtAddress())); + } + } + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private TradePresencesMessage(int id, List tradePresences) { @@ -70,49 +114,4 @@ public class TradePresencesMessage extends Message { return new TradePresencesMessage(id, tradePresences); } - @Override - protected synchronized byte[] toData() throws IOException { - if (this.cachedData != null) - return this.cachedData; - - // Shortcut in case we have no trade presences - if (this.tradePresences.isEmpty()) { - this.cachedData = Ints.toByteArray(0); - return this.cachedData; - } - - // How many of each timestamp - Map countByTimestamp = new HashMap<>(); - - for (TradePresenceData tradePresenceData : this.tradePresences) { - Long timestamp = tradePresenceData.getTimestamp(); - countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); - } - - // We should know exactly how many bytes to allocate now - int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) - + this.tradePresences.size() * (Transformer.PUBLIC_KEY_LENGTH + Transformer.SIGNATURE_LENGTH + Transformer.ADDRESS_LENGTH); - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - - bytes.write(Longs.toByteArray(timestamp)); - - for (TradePresenceData tradePresenceData : this.tradePresences) { - if (tradePresenceData.getTimestamp() == timestamp) { - bytes.write(tradePresenceData.getPublicKey()); - - bytes.write(tradePresenceData.getSignature()); - - bytes.write(Base58.decode(tradePresenceData.getAtAddress())); - } - } - } - - this.cachedData = bytes.toByteArray(); - return this.cachedData; - } - } diff --git a/src/main/java/org/qortal/network/message/TransactionMessage.java b/src/main/java/org/qortal/network/message/TransactionMessage.java index 6aa3c96f..51db6cf9 100644 --- a/src/main/java/org/qortal/network/message/TransactionMessage.java +++ b/src/main/java/org/qortal/network/message/TransactionMessage.java @@ -10,8 +10,11 @@ public class TransactionMessage extends Message { private TransactionData transactionData; - public TransactionMessage(TransactionData transactionData) { - this(-1, transactionData); + public TransactionMessage(TransactionData transactionData) throws TransformationException { + super(MessageType.TRANSACTION); + + this.dataBytes = TransactionTransformer.toBytes(transactionData); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private TransactionMessage(int id, TransactionData transactionData) { @@ -36,12 +39,4 @@ public class TransactionMessage extends Message { return new TransactionMessage(id, transactionData); } - @Override - protected byte[] toData() throws TransformationException { - if (this.transactionData == null) - return null; - - return TransactionTransformer.toBytes(this.transactionData); - } - } diff --git a/src/main/java/org/qortal/network/message/TransactionSignaturesMessage.java b/src/main/java/org/qortal/network/message/TransactionSignaturesMessage.java index 9d049e20..395d3f00 100644 --- a/src/main/java/org/qortal/network/message/TransactionSignaturesMessage.java +++ b/src/main/java/org/qortal/network/message/TransactionSignaturesMessage.java @@ -13,10 +13,24 @@ import com.google.common.primitives.Ints; public class TransactionSignaturesMessage extends Message { - private final List signatures; + private List signatures; public TransactionSignaturesMessage(List signatures) { - this(-1, signatures); + super(MessageType.TRANSACTION_SIGNATURES); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(signatures.size())); + + for (byte[] signature : signatures) + bytes.write(signature); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private TransactionSignaturesMessage(int id, List signatures) { @@ -45,16 +59,4 @@ public class TransactionSignaturesMessage extends Message { return new TransactionSignaturesMessage(id, signatures); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.signatures.size())); - - for (byte[] signature : this.signatures) - bytes.write(signature); - - return bytes.toByteArray(); - } - } From df3f16ccf1c68f89eec696f37beab7374c62341e Mon Sep 17 00:00:00 2001 From: catbref Date: Sun, 10 Apr 2022 16:08:47 +0100 Subject: [PATCH 11/14] EPC-fixes: Improve Network shutdown by exiting fast during broadcast and skipping callbacks during peer disconnect. --- src/main/java/org/qortal/network/Network.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index a82107ec..725d336f 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -132,6 +132,8 @@ public class Network { private String ourExternalIpAddress = null; private int ourExternalPort = Settings.getInstance().getListenPort(); + private volatile boolean isShuttingDown = false; + // Constructors private Network() { @@ -835,8 +837,6 @@ public class Network { } public void onDisconnect(Peer peer) { - // Notify Controller - Controller.getInstance().onPeerDisconnect(peer); if (peer.getConnectionEstablishedTime() > 0L) { LOGGER.debug("[{}] Disconnected from peer {}", peer.getPeerConnectionId(), peer); } else { @@ -846,6 +846,10 @@ public class Network { this.removeConnectedPeer(peer); this.channelsPendingWrite.remove(peer.getSocketChannel()); + if (this.isShuttingDown) + // No need to do any further processing, like re-enabling listen socket or notifying Controller + return; + if (getImmutableConnectedPeers().size() < maxPeers - 1 && serverSelectionKey.isValid() && (serverSelectionKey.interestOps() & SelectionKey.OP_ACCEPT) == 0) { @@ -856,6 +860,9 @@ public class Network { LOGGER.error("Failed to re-enable accepting of incoming connections: {}", e.getMessage()); } } + + // Notify Controller + Controller.getInstance().onPeerDisconnect(peer); } public void peerMisbehaved(Peer peer) { @@ -1454,6 +1461,9 @@ public class Network { public void broadcast(Function peerMessageBuilder) { for (Peer peer : getImmutableHandshakedPeers()) { + if (this.isShuttingDown) + return; + Message message = peerMessageBuilder.apply(peer); if (message == null) { @@ -1469,6 +1479,8 @@ public class Network { // Shutdown public void shutdown() { + this.isShuttingDown = true; + // Close listen socket to prevent more incoming connections if (this.serverChannel.isOpen()) { try { From c9b262046199aeb2eeb9a08fc2807b174e85726d Mon Sep 17 00:00:00 2001 From: catbref Date: Sun, 17 Apr 2022 19:37:28 +0100 Subject: [PATCH 12/14] EPC-fixes: fix constructing GET_ONLINE_ACCOUNTS_V2 message for case where onlineAccount args is empty list --- .../network/message/GetOnlineAccountsV2Message.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java index d3df3a1b..fe6b5d72 100644 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java @@ -29,6 +29,14 @@ public class GetOnlineAccountsV2Message extends Message { public GetOnlineAccountsV2Message(List onlineAccounts) { super(MessageType.GET_ONLINE_ACCOUNTS_V2); + // If we don't have ANY online accounts then it's an easier construction... + if (onlineAccounts.isEmpty()) { + // Always supply a number of accounts + this.dataBytes = Ints.toByteArray(0); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + return; + } + // How many of each timestamp Map countByTimestamp = new HashMap<>(); From 68412b49a16aa747805ce3c8ea57dbe417058f17 Mon Sep 17 00:00:00 2001 From: catbref Date: Sun, 17 Apr 2022 19:38:50 +0100 Subject: [PATCH 13/14] EPC-fixes: use bindAddress from Settings for outgoing peer connections, not just listen socket --- src/main/java/org/qortal/network/Peer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index 80b888d2..dbb03fda 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -413,6 +413,8 @@ public class Peer { this.isLocal = isAddressLocal(this.resolvedAddress.getAddress()); this.socketChannel = SocketChannel.open(); + InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress()); + this.socketChannel.socket().bind(new InetSocketAddress(bindAddr, 0)); this.socketChannel.socket().connect(resolvedAddress, CONNECT_TIMEOUT); } catch (SocketTimeoutException e) { LOGGER.trace("[{}] Connection timed out to peer {}", this.peerConnectionId, this); From 3e622f718586f4fffd0d8017498a73aced00709a Mon Sep 17 00:00:00 2001 From: catbref Date: Mon, 18 Apr 2022 14:33:05 +0100 Subject: [PATCH 14/14] EPC-fixes: catch CancelledKeyExceptions thrown in short window between nextSelectionKey.isValid() and nextSelectionKey.isXXXable() calls --- src/main/java/org/qortal/network/Network.java | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 725d336f..9c797901 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -590,32 +590,41 @@ public class Network { SelectableChannel socketChannel = nextSelectionKey.channel(); - if (nextSelectionKey.isReadable()) { - clearInterestOps(nextSelectionKey, SelectionKey.OP_READ); - Peer peer = getPeerFromChannel((SocketChannel) socketChannel); - if (peer == null) - return null; + try { + if (nextSelectionKey.isReadable()) { + clearInterestOps(nextSelectionKey, SelectionKey.OP_READ); + Peer peer = getPeerFromChannel((SocketChannel) socketChannel); + if (peer == null) + return null; - return new ChannelReadTask((SocketChannel) socketChannel, peer); - } + return new ChannelReadTask((SocketChannel) socketChannel, peer); + } - if (nextSelectionKey.isWritable()) { - clearInterestOps(nextSelectionKey, SelectionKey.OP_WRITE); - Peer peer = getPeerFromChannel((SocketChannel) socketChannel); - if (peer == null) - return null; + if (nextSelectionKey.isWritable()) { + clearInterestOps(nextSelectionKey, SelectionKey.OP_WRITE); + Peer peer = getPeerFromChannel((SocketChannel) socketChannel); + if (peer == null) + return null; - // Any thread that queues a message to send can set OP_WRITE, - // but we only allow one pending/active ChannelWriteTask per Peer - if (!channelsPendingWrite.add(socketChannel)) - return null; + // Any thread that queues a message to send can set OP_WRITE, + // but we only allow one pending/active ChannelWriteTask per Peer + if (!channelsPendingWrite.add(socketChannel)) + return null; - return new ChannelWriteTask((SocketChannel) socketChannel, peer); - } + return new ChannelWriteTask((SocketChannel) socketChannel, peer); + } - if (nextSelectionKey.isAcceptable()) { - clearInterestOps(nextSelectionKey, SelectionKey.OP_ACCEPT); - return new ChannelAcceptTask((ServerSocketChannel) socketChannel); + if (nextSelectionKey.isAcceptable()) { + clearInterestOps(nextSelectionKey, SelectionKey.OP_ACCEPT); + return new ChannelAcceptTask((ServerSocketChannel) socketChannel); + } + } catch (CancelledKeyException e) { + /* + * Sometimes nextSelectionKey is cancelled / becomes invalid between the isValid() test at line 586 + * and later calls to isReadable() / isWritable() / isAcceptable() which themselves call isValid()! + * Those isXXXable() calls could throw CancelledKeyException, so we catch it here and return null. + */ + return null; } }