From e505067759fd39dd8c9cfadbecd2d2c7d3e0a60e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 21 Feb 2021 12:02:26 +0000 Subject: [PATCH 01/54] Added support for Native SegWit (Bech32) addresses, which have the prefixes "ltc1" and "bc1". Bitcoinj supports these automatically, as long as fromString() is used instead of fromBase58(), which is already the case. Tested on LTC mainnet, and BTC testnet. --- .../org/qortal/api/resource/CrossChainBitcoinResource.java | 2 +- .../org/qortal/api/resource/CrossChainLitecoinResource.java | 2 +- src/main/java/org/qortal/crosschain/Bitcoiny.java | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java index 445d853e..efffc3b0 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java @@ -113,7 +113,7 @@ public class CrossChainBitcoinResource { @Path("/send") @Operation( summary = "Sends BTC from hierarchical, deterministic BIP32 wallet to specific address", - description = "Currently only supports 'legacy' P2PKH Bitcoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", + description = "Currently supports 'legacy' P2PKH Bitcoin addresses and Native SegWit (P2WPKH) addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", requestBody = @RequestBody( required = true, content = @Content( diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java index 9c841045..13803143 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java @@ -113,7 +113,7 @@ public class CrossChainLitecoinResource { @Path("/send") @Operation( summary = "Sends LTC from hierarchical, deterministic BIP32 wallet to specific address", - description = "Currently only supports 'legacy' P2PKH Litecoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", + description = "Currently supports 'legacy' P2PKH Litecoin addresses and Native SegWit (P2WPKH) addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", requestBody = @RequestBody( required = true, content = @Content( diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index c3fecb4d..0fe508a2 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -98,8 +98,9 @@ public abstract class Bitcoiny implements ForeignBlockchain { try { ScriptType addressType = Address.fromString(this.params, address).getOutputScriptType(); - return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH; + return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH || addressType == ScriptType.P2WPKH; } catch (AddressFormatException e) { + LOGGER.error(String.format("Unrecognised address format: %s", address)); return false; } } From 244d4f78e2136c497ae9a0a3cdd15b04a27df7bd Mon Sep 17 00:00:00 2001 From: catbref Date: Sun, 23 Jan 2022 16:24:10 +0000 Subject: [PATCH 02/54] New network messages ONLINE_ACCOUNTS_V2 and GET_ONLINE_ACCOUNTS_V2. Increased GetOnlineAccountsMessage.MAX_ACCOUNT_COUNT from 1000 to 5000. The V2 versions are more efficiently encoded and also cache the payload bytes which reduces CPU when sending to multiple peers. Serialization / deserialization unit tests included. Tentative V2 message activation set at core version 3.1.2 see Controller.ONLINE_ACCOUNTS_V2_PEER_VERSION --- .../org/qortal/controller/Controller.java | 77 ++++++++++- .../message/GetOnlineAccountsMessage.java | 2 +- .../message/GetOnlineAccountsV2Message.java | 117 +++++++++++++++++ .../org/qortal/network/message/Message.java | 2 + .../message/OnlineAccountsV2Message.java | 124 ++++++++++++++++++ .../test/network/OnlineAccountsTests.java | 114 ++++++++++++++++ 6 files changed, 430 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java create mode 100644 src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java create mode 100644 src/test/java/org/qortal/test/network/OnlineAccountsTests.java diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 2bfc80c2..c19c57f7 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -109,6 +109,8 @@ public class Controller extends Thread { private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000L); /** How many (latest) blocks' worth of online accounts we cache */ private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 2; + private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300010002L; + private static volatile boolean isStopping = false; private static BlockMinter blockMinter = null; @@ -1376,6 +1378,14 @@ public class Controller extends Thread { onNetworkOnlineAccountsMessage(peer, message); break; + case GET_ONLINE_ACCOUNTS_V2: + onNetworkGetOnlineAccountsV2Message(peer, message); + break; + + case ONLINE_ACCOUNTS_V2: + onNetworkOnlineAccountsV2Message(peer, message); + break; + case GET_ARBITRARY_DATA: // Not currently supported break; @@ -1808,6 +1818,53 @@ public class Controller extends Thread { } } + private void onNetworkGetOnlineAccountsV2Message(Peer peer, Message message) { + GetOnlineAccountsV2Message getOnlineAccountsMessage = (GetOnlineAccountsV2Message) message; + + List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); + + // Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts + List accountsToSend; + synchronized (this.onlineAccounts) { + accountsToSend = new ArrayList<>(this.onlineAccounts); + } + + Iterator iterator = accountsToSend.iterator(); + + SEND_ITERATOR: + while (iterator.hasNext()) { + OnlineAccountData onlineAccountData = iterator.next(); + + for (int i = 0; i < excludeAccounts.size(); ++i) { + OnlineAccountData excludeAccountData = excludeAccounts.get(i); + + if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) { + iterator.remove(); + continue SEND_ITERATOR; + } + } + } + + Message onlineAccountsMessage = new OnlineAccountsV2Message(accountsToSend); + peer.sendMessage(onlineAccountsMessage); + + LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer)); + } + + private void onNetworkOnlineAccountsV2Message(Peer peer, Message message) { + OnlineAccountsV2Message onlineAccountsMessage = (OnlineAccountsV2Message) message; + + List peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts(); + LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer)); + + try (final Repository repository = RepositoryManager.getRepository()) { + for (OnlineAccountData onlineAccountData : peersOnlineAccounts) + this.verifyAndAddAccount(repository, onlineAccountData); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e); + } + } + // Utilities private void verifyAndAddAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException { @@ -1919,11 +1976,17 @@ public class Controller extends Thread { // Request data from other peers? if ((this.onlineAccountsTasksTimestamp % ONLINE_ACCOUNTS_BROADCAST_INTERVAL) < ONLINE_ACCOUNTS_TASKS_INTERVAL) { - Message message; + List safeOnlineAccounts; synchronized (this.onlineAccounts) { - message = new GetOnlineAccountsMessage(this.onlineAccounts); + safeOnlineAccounts = new ArrayList<>(this.onlineAccounts); } - Network.getInstance().broadcast(peer -> message); + + Message messageV1 = new GetOnlineAccountsMessage(safeOnlineAccounts); + Message messageV2 = new GetOnlineAccountsV2Message(safeOnlineAccounts); + + Network.getInstance().broadcast(peer -> + peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1 + ); } // Refresh our online accounts signatures? @@ -2010,8 +2073,12 @@ public class Controller extends Thread { if (!hasInfoChanged) return; - Message message = new OnlineAccountsMessage(ourOnlineAccounts); - Network.getInstance().broadcast(peer -> message); + Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts); + Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts); + + Network.getInstance().broadcast(peer -> + peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1 + ); LOGGER.trace(()-> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp)); } diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java index 93f782df..23c21bc5 100644 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java @@ -15,7 +15,7 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; public class GetOnlineAccountsMessage extends Message { - private static final int MAX_ACCOUNT_COUNT = 1000; + private static final int MAX_ACCOUNT_COUNT = 5000; private List onlineAccounts; diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java new file mode 100644 index 00000000..709f9782 --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java @@ -0,0 +1,117 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.qortal.data.network.OnlineAccountData; +import org.qortal.transform.Transformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * For requesting online accounts info from remote peer, given our list of online accounts. + * + * Different format to V1: + * V1 is: number of entries, then timestamp + pubkey for each entry + * V2 is: groups of: number of entries, timestamp, then pubkey for each entry + * + * Also V2 only builds online accounts message once! + */ +public class GetOnlineAccountsV2Message extends Message { + private List onlineAccounts; + private byte[] cachedData; + + public GetOnlineAccountsV2Message(List onlineAccounts) { + this(-1, onlineAccounts); + } + + private GetOnlineAccountsV2Message(int id, List onlineAccounts) { + super(id, MessageType.GET_ONLINE_ACCOUNTS_V2); + + this.onlineAccounts = onlineAccounts; + } + + public List getOnlineAccounts() { + return this.onlineAccounts; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + int accountCount = bytes.getInt(); + + List onlineAccounts = new ArrayList<>(accountCount); + + while (accountCount > 0) { + long timestamp = bytes.getLong(); + + for (int i = 0; i < accountCount; ++i) { + byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + bytes.get(publicKey); + + onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey)); + } + + if (bytes.hasRemaining()) { + accountCount = bytes.getInt(); + } else { + // we've finished + accountCount = 0; + } + } + + return new GetOnlineAccountsV2Message(id, onlineAccounts); + } + + @Override + protected synchronized byte[] toData() { + if (this.cachedData != null) + return this.cachedData; + + // Shortcut in case we have no online accounts + if (this.onlineAccounts.isEmpty()) { + this.cachedData = Ints.toByteArray(0); + return this.cachedData; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (int i = 0; i < this.onlineAccounts.size(); ++i) { + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + Long timestamp = onlineAccountData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + this.onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH; + + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (int i = 0; i < this.onlineAccounts.size(); ++i) { + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + + if (onlineAccountData.getTimestamp() == timestamp) + bytes.write(onlineAccountData.getPublicKey()); + } + } + + this.cachedData = bytes.toByteArray(); + return this.cachedData; + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/network/message/Message.java b/src/main/java/org/qortal/network/message/Message.java index c7657493..6c89a0dd 100644 --- a/src/main/java/org/qortal/network/message/Message.java +++ b/src/main/java/org/qortal/network/message/Message.java @@ -78,6 +78,8 @@ public abstract class Message { ONLINE_ACCOUNTS(80), GET_ONLINE_ACCOUNTS(81), + ONLINE_ACCOUNTS_V2(82), + GET_ONLINE_ACCOUNTS_V2(83), ARBITRARY_DATA(90), GET_ARBITRARY_DATA(91), diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java new file mode 100644 index 00000000..f0fce81e --- /dev/null +++ b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java @@ -0,0 +1,124 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.qortal.data.network.OnlineAccountData; +import org.qortal.transform.Transformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * For sending online accounts info to remote peer. + * + * Different format to V1: + * V1 is: number of entries, then timestamp + sig + pubkey for each entry + * V2 is: groups of: number of entries, timestamp, then sig + pubkey for each entry + * + * Also V2 only builds online accounts message once! + */ +public class OnlineAccountsV2Message extends Message { + private List onlineAccounts; + private byte[] cachedData; + + public OnlineAccountsV2Message(List onlineAccounts) { + this(-1, onlineAccounts); + } + + private OnlineAccountsV2Message(int id, List onlineAccounts) { + super(id, MessageType.ONLINE_ACCOUNTS_V2); + + this.onlineAccounts = onlineAccounts; + } + + public List getOnlineAccounts() { + return this.onlineAccounts; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + int accountCount = bytes.getInt(); + + List onlineAccounts = new ArrayList<>(accountCount); + + while (accountCount > 0) { + long timestamp = bytes.getLong(); + + for (int i = 0; i < accountCount; ++i) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; + bytes.get(signature); + + byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + bytes.get(publicKey); + + onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey)); + } + + if (bytes.hasRemaining()) { + accountCount = bytes.getInt(); + } else { + // we've finished + accountCount = 0; + } + } + + return new OnlineAccountsV2Message(id, onlineAccounts); + } + + @Override + protected synchronized byte[] toData() { + if (this.cachedData != null) + return this.cachedData; + + // Shortcut in case we have no online accounts + if (this.onlineAccounts.isEmpty()) { + this.cachedData = Ints.toByteArray(0); + return this.cachedData; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (int i = 0; i < this.onlineAccounts.size(); ++i) { + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + Long timestamp = onlineAccountData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + this.onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH); + + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (int i = 0; i < this.onlineAccounts.size(); ++i) { + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + + if (onlineAccountData.getTimestamp() == timestamp) { + bytes.write(onlineAccountData.getSignature()); + + bytes.write(onlineAccountData.getPublicKey()); + } + } + } + + this.cachedData = bytes.toByteArray(); + return this.cachedData; + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java new file mode 100644 index 00000000..b1c5ec4f --- /dev/null +++ b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java @@ -0,0 +1,114 @@ +package org.qortal.test.network; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.junit.Test; +import org.qortal.data.network.OnlineAccountData; +import org.qortal.network.message.*; +import org.qortal.transform.Transformer; + +import java.nio.ByteBuffer; +import java.security.Security; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class OnlineAccountsTests { + + private static final Random RANDOM = new Random(); + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + } + + + @Test + public void testGetOnlineAccountsV2() throws Message.MessageException { + List onlineAccountsOut = generateOnlineAccounts(false); + + Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut); + + byte[] messageBytes = messageOut.toBytes(); + ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes); + + GetOnlineAccountsV2Message messageIn = (GetOnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer); + + List onlineAccountsIn = messageIn.getOnlineAccounts(); + + assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size()); + assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut)); + + Message oldMessageOut = new GetOnlineAccountsMessage(onlineAccountsOut); + byte[] oldMessageBytes = oldMessageOut.toBytes(); + + long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count(); + + System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d", + onlineAccountsOut.size(), + numTimestamps, + numTimestamps != 1 ? "s" : "", + oldMessageBytes.length, + messageBytes.length)); + } + + @Test + public void testOnlineAccountsV2() throws Message.MessageException { + List onlineAccountsOut = generateOnlineAccounts(true); + + Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut); + + byte[] messageBytes = messageOut.toBytes(); + ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes); + + OnlineAccountsV2Message messageIn = (OnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer); + + List onlineAccountsIn = messageIn.getOnlineAccounts(); + + assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size()); + assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut)); + + Message oldMessageOut = new OnlineAccountsMessage(onlineAccountsOut); + byte[] oldMessageBytes = oldMessageOut.toBytes(); + + long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count(); + + System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d", + onlineAccountsOut.size(), + numTimestamps, + numTimestamps != 1 ? "s" : "", + oldMessageBytes.length, + messageBytes.length)); + } + + private List generateOnlineAccounts(boolean withSignatures) { + List onlineAccounts = new ArrayList<>(); + + int numTimestamps = RANDOM.nextInt(2) + 1; // 1 or 2 + + for (int t = 0; t < numTimestamps; ++t) { + int numAccounts = RANDOM.nextInt(3000); + + for (int a = 0; a < numAccounts; ++a) { + byte[] sig = null; + if (withSignatures) { + sig = new byte[Transformer.SIGNATURE_LENGTH]; + RANDOM.nextBytes(sig); + } + + byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + RANDOM.nextBytes(pubkey); + + onlineAccounts.add(new OnlineAccountData(t << 32, sig, pubkey)); + } + } + + return onlineAccounts; + } + +} From 58f11489db56347f5cf4691b7b6d18d8b77ea70c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 13 Feb 2022 20:32:19 +0000 Subject: [PATCH 03/54] Bump version to 3.1.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 798d68ea..df640288 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.0.4 + 3.1.0 jar true From 538ac30b4eebe77db98c402bef48011b6ef9b62e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 14 Feb 2022 19:33:36 +0000 Subject: [PATCH 04/54] Request only the missing hashes, not all of them. --- .../ArbitraryDataFileListManager.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index 46c2ff15..87620128 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -266,18 +266,16 @@ public class ArbitraryDataFileListManager { List handshakedPeers = Network.getInstance().getHandshakedPeers(); List missingHashes = null; -// // TODO: uncomment after GetArbitraryDataFileListMessage updates are deployed -// // Find hashes that we are missing -// try { -// ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); -// arbitraryDataFile.setMetadataHash(metadataHash); -// missingHashes = arbitraryDataFile.missingHashes(); -// } catch (DataException e) { -// // Leave missingHashes as null, so that all hashes are requested -// } -// int hashCount = missingHashes != null ? missingHashes.size() : 0; + // Find hashes that we are missing + try { + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + missingHashes = arbitraryDataFile.missingHashes(); + } catch (DataException e) { + // Leave missingHashes as null, so that all hashes are requested + } + int hashCount = missingHashes != null ? missingHashes.size() : 0; - int hashCount = 0; LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size())); // Build request From 43791f00aa338d410e5dbab588453fd9b009cf34 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 14 Feb 2022 19:33:58 +0000 Subject: [PATCH 05/54] Wait 2 minutes on node startup before trying to fetch followed QDN data. --- .../org/qortal/controller/arbitrary/ArbitraryDataManager.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index f07f0669..8dc9c0e2 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -80,6 +80,9 @@ public class ArbitraryDataManager extends Thread { Thread.currentThread().setName("Arbitrary Data Manager"); try { + // Wait for node to finish starting up and making connections + Thread.sleep(2 * 60 * 1000L); + while (!isStopping) { Thread.sleep(2000); From 6c1c814aca24ab19f87b7cf0c930006119389986 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 14 Feb 2022 20:15:37 +0000 Subject: [PATCH 06/54] Updated AdvancedInstaller project for v3.1.0 --- WindowsInstaller/Qortal.aip | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/WindowsInstaller/Qortal.aip b/WindowsInstaller/Qortal.aip index 59da9519..46f8260c 100755 --- a/WindowsInstaller/Qortal.aip +++ b/WindowsInstaller/Qortal.aip @@ -17,10 +17,10 @@ - + - + @@ -212,7 +212,7 @@ - + From fd0a6ec71fc5cd050c2cff2e6a7f94c7130521d8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 14 Feb 2022 22:53:30 +0000 Subject: [PATCH 07/54] Fix for invalid balance (and transaction amount) when there are no outputs relating to this wallet. --- src/main/java/org/qortal/crosschain/Bitcoiny.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index a5a1ab12..b29d9fe3 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -470,6 +470,8 @@ public abstract class Bitcoiny implements ForeignBlockchain { List inputs = new ArrayList<>(); List outputs = new ArrayList<>(); + boolean anyOutputAddressInWallet = false; + for (BitcoinyTransaction.Input input : t.inputs) { try { BitcoinyTransaction t2 = getTransaction(input.outputTxHash); @@ -502,6 +504,7 @@ public abstract class Bitcoiny implements ForeignBlockchain { amount += output.value; } addressInWallet = true; + anyOutputAddressInWallet = true; } outputs.add(new SimpleTransaction.Output(address, output.value, addressInWallet)); } @@ -510,6 +513,13 @@ public abstract class Bitcoiny implements ForeignBlockchain { } } long fee = totalInputAmount - totalOutputAmount; + + if (!anyOutputAddressInWallet) { + // No outputs relate to this wallet - check if any inputs did (which is signified by a positive total) + if (total > 0) { + amount = total * -1; + } + } return new SimpleTransaction(t.txHash, t.timestamp, amount, fee, inputs, outputs); } From 6275ac2b81cf9215e5fbf5655e62a1cb31e1b04b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 14 Feb 2022 22:58:37 +0000 Subject: [PATCH 08/54] Increased numberOfAdditionalBatchesToSearch from 5 to 7. This is the equivalent of increasing the max address gap from 15 to 21. The electrum standalone wallet uses 20, so this should be the most we will ever need. --- src/main/java/org/qortal/crosschain/Bitcoiny.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index b29d9fe3..05d3aaa9 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -404,7 +404,7 @@ public abstract class Bitcoiny implements ForeignBlockchain { Set keySet = new HashSet<>(); // Set the number of consecutive empty batches required before giving up - final int numberOfAdditionalBatchesToSearch = 5; + final int numberOfAdditionalBatchesToSearch = 7; int unusedCounter = 0; int ki = 0; From 6ee395ed12ef11970c78f610f3f129f70cd7900e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 15 Feb 2022 19:10:20 +0000 Subject: [PATCH 09/54] Stop bulk arbitrary signature broadcasts, as they were creating a lot of network traffic, and are in the process of being replaced with a better method. --- src/main/java/org/qortal/network/Network.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 2a25864a..c9ae3b7a 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -1178,7 +1178,7 @@ public class Network { public void onExternalIpUpdate(String ipAddress) { LOGGER.info("External IP address updated to {}", ipAddress); - ArbitraryDataManager.getInstance().broadcastHostedSignatureList(); + //ArbitraryDataManager.getInstance().broadcastHostedSignatureList(); } From c90c287601563dc8cee05d6fecfaa30e778a35b6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 15 Feb 2022 19:38:30 +0000 Subject: [PATCH 10/54] Increased ARBITRARY_REQUEST_TIMEOUT from 10 to 12 seconds, as some were coming back around 9-10 seconds later. --- .../org/qortal/controller/arbitrary/ArbitraryDataManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 8dc9c0e2..acde16eb 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -38,7 +38,7 @@ public class ArbitraryDataManager extends Thread { private int powDifficulty = 14; // Must not be final, as unit tests need to reduce this value /** Request timeout when transferring arbitrary data */ - public static final long ARBITRARY_REQUEST_TIMEOUT = 10 * 1000L; // ms + public static final long ARBITRARY_REQUEST_TIMEOUT = 12 * 1000L; // ms /** Maximum time to hold information about an in-progress relay */ public static final long ARBITRARY_RELAY_TIMEOUT = 60 * 1000L; // ms From 85b3278c8a4d77aeb03b7b967e1f482904f0401c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 15 Feb 2022 19:39:26 +0000 Subject: [PATCH 11/54] Don't throttle the arbitrary data file request threads when there are items to process. --- .../controller/arbitrary/ArbitraryDataFileRequestThread.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java index 5f491411..0c2834d0 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -31,8 +31,6 @@ public class ArbitraryDataFileRequestThread implements Runnable { try { while (!Controller.isStopping()) { - Thread.sleep(1000); - Long now = NTP.getTime(); this.processFileHashes(now); } @@ -41,7 +39,7 @@ public class ArbitraryDataFileRequestThread implements Runnable { } } - private void processFileHashes(Long now) { + private void processFileHashes(Long now) throws InterruptedException { if (Controller.isStopping()) { return; } @@ -91,6 +89,7 @@ public class ArbitraryDataFileRequestThread implements Runnable { if (!shouldProcess) { // Nothing to do + Thread.sleep(1000L); return; } From 605498237936d84315b5df4f7a9f2b90e44e308c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 15 Feb 2022 20:02:21 +0000 Subject: [PATCH 12/54] Bump version to 3.1.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index df640288..1d7eebeb 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.1.0 + 3.1.1 jar true From d1a7e734dcf72c9148fc178592d1692dac91e3f9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 15 Feb 2022 23:36:38 +0000 Subject: [PATCH 13/54] Updated AdvancedInstaller project for v3.1.1 --- WindowsInstaller/Qortal.aip | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/WindowsInstaller/Qortal.aip b/WindowsInstaller/Qortal.aip index 46f8260c..f69f0682 100755 --- a/WindowsInstaller/Qortal.aip +++ b/WindowsInstaller/Qortal.aip @@ -17,10 +17,10 @@ - + - + @@ -212,7 +212,7 @@ - + From c90c3a183e67f1e7e777900e543d0c43f40d1ffd Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Feb 2022 14:50:29 +0000 Subject: [PATCH 14/54] Block peers below 3.1.0 --- src/main/java/org/qortal/network/Handshake.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/qortal/network/Handshake.java b/src/main/java/org/qortal/network/Handshake.java index d88654cf..cdcff1d7 100644 --- a/src/main/java/org/qortal/network/Handshake.java +++ b/src/main/java/org/qortal/network/Handshake.java @@ -74,6 +74,12 @@ public enum Handshake { peer.setPeersConnectionTimestamp(peersConnectionTimestamp); peer.setPeersVersion(versionString, version); + // Ensure the peer is running at least the version specified in MIN_PEER_VERSION + if (peer.isAtLeastVersion(MIN_PEER_VERSION) == false) { + LOGGER.debug(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString)); + return null; + } + if (Settings.getInstance().getAllowConnectionsWithOlderPeerVersions() == false) { // Ensure the peer is running at least the minimum version allowed for connections final String minPeerVersion = Settings.getInstance().getMinPeerVersion(); @@ -258,6 +264,9 @@ public enum Handshake { private static final long PEER_VERSION_131 = 0x0100030001L; + /** Minimum peer version that we are allowed to communicate with */ + private static final String MIN_PEER_VERSION = "3.1.0"; + private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes private static final int POW_DIFFICULTY_PRE_131 = 8; // leading zero bits // Can always be made harder in the future... From 3e505481fedf5272d1365a3c7b8dfa288ff1430b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Feb 2022 14:50:46 +0000 Subject: [PATCH 15/54] Default minPeerVersion also increased to 3.1.0 --- src/main/java/org/qortal/settings/Settings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 45f89697..a3e03028 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -195,7 +195,7 @@ public class Settings { private int maxRetries = 2; /** Minimum peer version number required in order to sync with them */ - private String minPeerVersion = "3.0.1"; + private String minPeerVersion = "3.1.0"; /** Whether to allow connections with peers below minPeerVersion * If true, we won't sync with them but they can still sync with us, and will show in the peers list * If false, sync will be blocked both ways, and they will not appear in the peers list */ From 464ce66fd53cd67babdf0ac5fcdd317846e9df5c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Feb 2022 15:09:50 +0000 Subject: [PATCH 16/54] Moved deletion retry code into ArbitraryDataFile --- .../org/qortal/arbitrary/ArbitraryDataFile.java | 15 +++++++++++++++ .../arbitrary/ArbitraryDataFileManager.java | 11 +---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 2d7346ea..6b29de8d 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -366,6 +366,21 @@ public class ArbitraryDataFile { return false; } + public boolean delete(int attempts) { + // Keep trying to delete the data until it is deleted, or we reach 10 attempts + for (int i=0; i Date: Fri, 18 Feb 2022 15:35:37 +0000 Subject: [PATCH 17/54] Log if we're unable to process the received file. --- .../org/qortal/network/message/ArbitraryDataFileMessage.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java index d87e9685..b9f24e29 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java @@ -1,6 +1,8 @@ package org.qortal.network.message; import com.google.common.primitives.Ints; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.repository.DataException; import org.qortal.transform.Transformer; @@ -12,6 +14,8 @@ import java.nio.ByteBuffer; public class ArbitraryDataFileMessage extends Message { + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileMessage.class); + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; private final byte[] signature; @@ -52,6 +56,7 @@ public class ArbitraryDataFileMessage extends Message { return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile); } catch (DataException e) { + LOGGER.info("Unable to process received file: {}", e.getMessage()); return null; } } From 5aac2dc9df9022589158dd7cedbbbe1e5f2dc717 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Feb 2022 18:34:00 +0000 Subject: [PATCH 18/54] ONLINE_ACCOUNTS_V2_PEER_VERSION set to 3.2.0 --- src/main/java/org/qortal/controller/Controller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 974567f4..2bf7d973 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -105,7 +105,7 @@ public class Controller extends Thread { private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000L); /** How many (latest) blocks' worth of online accounts we cache */ private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 2; - private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300010002L; + private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L; private static volatile boolean isStopping = false; From fcdd85af6cb9421cf6bb86ba1c7ea6cf2083b538 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 8 Feb 2022 18:27:44 +0000 Subject: [PATCH 19/54] Try a lookahead size of 20 (instead of 3) when asking Bitcoinj for the balance. --- src/main/java/org/qortal/crosschain/Bitcoiny.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index 05d3aaa9..0d26c397 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -58,9 +58,14 @@ public abstract class Bitcoiny implements ForeignBlockchain { * i.e. keys with transactions but with no unspent outputs. */ protected final Set spentKeys = Collections.synchronizedSet(new HashSet<>()); - /** How many bitcoinj wallet keys to generate in each batch. */ + /** How many wallet keys to generate in each batch. */ private static final int WALLET_KEY_LOOKAHEAD_INCREMENT = 3; + /** How many wallet keys to generate in each batch when using bitcoinj as the data provider. + * We must use a higher value here since we are unable to request multiple batches of keys. + * Without this, the bitcoinj balance (or other data) can be missing transactions. */ + private static final int WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ = 20; + /** Byte offset into raw block headers to block timestamp. */ private static final int TIMESTAMP_OFFSET = 4 + 32 + 32; @@ -612,7 +617,7 @@ public abstract class Bitcoiny implements ForeignBlockchain { this.keyChain = this.wallet.getActiveKeyChain(); // Set up wallet's key chain - this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ); this.keyChain.maybeLookAhead(); } From 35b0a8581862c5e19138ebd2289a13f888ccd4a7 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Feb 2022 18:42:06 +0000 Subject: [PATCH 20/54] Increased WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ to 50 --- src/main/java/org/qortal/crosschain/Bitcoiny.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index 0d26c397..cce20c84 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -61,10 +61,10 @@ public abstract class Bitcoiny implements ForeignBlockchain { /** How many wallet keys to generate in each batch. */ private static final int WALLET_KEY_LOOKAHEAD_INCREMENT = 3; - /** How many wallet keys to generate in each batch when using bitcoinj as the data provider. + /** How many wallet keys to generate when using bitcoinj as the data provider. * We must use a higher value here since we are unable to request multiple batches of keys. - * Without this, the bitcoinj balance (or other data) can be missing transactions. */ - private static final int WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ = 20; + * Without this, the bitcoinj state can be missing transactions, causing errors such as "insufficient balance". */ + private static final int WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ = 50; /** Byte offset into raw block headers to block timestamp. */ private static final int TIMESTAMP_OFFSET = 4 + 32 + 32; From 5842b1272db5de205534d9d9c0e9baad420dfada Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Feb 2022 20:05:14 +0000 Subject: [PATCH 21/54] Add WaifUPnP-1.1 jar to project. For future reference, the command used was: mvn install:install-file -Dfile=/Users/user/Downloads/waifupnp-1.1/WaifUPnP.jar -DgroupId=com.dosse -DartifactId=WaifUPnP -Dversion=1.1 -Dpackaging=jar -DlocalRepositoryPath=lib --- lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.jar | Bin 0 -> 9934 bytes lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom | 9 +++++++++ lib/com/dosse/WaifUPnP/maven-metadata-local.xml | 12 ++++++++++++ pom.xml | 7 +++++++ 4 files changed, 28 insertions(+) create mode 100644 lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.jar create mode 100644 lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom create mode 100644 lib/com/dosse/WaifUPnP/maven-metadata-local.xml diff --git a/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.jar b/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.jar new file mode 100644 index 0000000000000000000000000000000000000000..10a92c071055d96b6648603ab4fa192fed967cb3 GIT binary patch literal 9934 zcmaia19)Z0+I4K(wrzBhj!$g6qmJD{cWkF)`^0wAvC*+@+xmOHnYnZCT>Q1qKJ}dE zS!>m<+O@0R{hp;L3l0GX0s;d9V)AhT0P;@-76bxBPFz)lQCePt`E3{kMDY(O42bbB zXvD&3Ch!-0{@01|>-+~)PDEZ>LR>|aNlrp8q8h221Vh{n?QJuX??aeueHvGk8nnsE zsI`X#8XSa0-B;!o3&$eQvt~j*r=o3kV40OLC1jP<%1S}3ZK?Lc-Msgd<6|xSA zY#4>V^fd{ysAbyTBF0pqM@-B;e_;U9R3W0xT)ob{MN7r;uX?)Tw88c`)y(0?{(XGF z=Z|8I6K_9XSAX4{{Y-(h_siF7mv1qxMk3Zfmwz6)XRczL#_so}GR(i#rUd_ROuw2RtQ*CSb zyuiPyrG35Lo_}ydzlXXYewYC9@qeG%YXsE|zpsC^LvK912e)|}=q02(5<~BKfdinb z(A6EY=1)aKwT(!$Ej=Kxz9fgc$+5}-+4_iMzG;F26InrV>9^}}?WBGtz=D1>&86jh z$}k-^Rgq_<%y(4aGz}vjD{-s^aPDxZ$lsf!r02+$)0(HU$+eoNI5yDkCp%WqPRjw= z&4OreO9lWLnY&3)Ddp`fjETHKqXm6z6EW9d2Z`6K=Vw(Zed-jV+t)jz>?jpfcCVo zjLoc`HYu&9KLJUFPfn{e__BiZ7~v4jeLF3eRfnb7#>v^5BfZXPQ=My{`Rs6LVete^ zgv8psq2I)H|BmC)kI8^6^#svvhwajoS?i@*l`Tn{a<0M>{Q+-#$h|_ErT%P~k~)UZ zBqO_qcUH{TWO+-v)jRuZUdCaBEr3TS-*~dO#e%A0TR^EsH#Xv~_IMFbE{5g6{?nJ9 z~G18C2Ro-ZASjTVkYqaO>qLF~-UK?uBprE)K8{Xeg7Ewred|OsE5JQiXgqJa8Gx$X`D9R2u zV$o8~KOE6&omkGX)Wc;UOF7ghq1CXznDxGpE9La1p~;|>Go_AwOqHgzWHv(AzM-ej ztixi|#h;mQ?BjFom)tyYPc|++)f74L>G2IdBp6(#)Gr7mBTmg6pfiA$(-76h0Vh`p z`#YHVUAD<`S1Noto`D6KC<~?n{Y(xto?dGr0r=s=HFq-N@5-q(u5W|0)Xn2D^oY%2 z%{?)txRZV*QSNqZsrZA6IcY*y-{c*WaNkSSAml4EoZ4nvv{|ZcC7O#Xs1!y>R@0mp z1E@)x$% z`I=-9)K3CPAzae1XqHFGN>HCKiLb-O){yAtT7fi?VKYQ^SYUNp%BPd$GC!v?MePty zYaUJsdpNH@6_;=YpfTYDTE{Q?)@j?`&+H*idQS)GnosslQl{YqL) z$ZSaO%2J||(v2SLEY=f#fR`=sXJBOFO40O`>Co#E&;kK{Dbi?lE4{G)qBx^xpg9Ph zIA#K6t`9KCU@_!>rPT$b9*W}6(pe8y$ z6{bpP=Rg+~ZO+~D>Sq;RPY;=Je~^JvcAzd@qVevQR%^oI%=F@@tAM$7{Mqb1(jSX3QhxLw$Q4AqqqE6|IOlkC1? zsVSvu`&6+(bSEXDG*KcJ-i$y{h>S1gJSq|%5-j>EuM{RzHUq0jf3D&w!>2pEB_TI` z)_d|~She5PHnogA7~4=Sj`bj>Zg{l!+2ea}E#vDULHp*!UT%tR1APUly2>Z&?}okc zf_EDwa=HfdeEu#rYOaPRb16l6$J#n0YpWd}g&ctrrEts>T;mx`@B;*$ibH` z1!hdr@DdPhJtqC(r~;M{ozm~IY}+-iwvl|yX*!nsPs)PhJ$#KE^;RTFM2g>!=B$$; zqc1P}rIvB!FjPhY&si%Y3v4TJ#up}iu$hs*3Scs2C{ZNkvC}n}9f)X)Yn2bhg$_~X z%1~hu@UJMLQ%B?&RxTYQU03gfG4D<&(0E2{Io=JPc&V*k+S|SgR*E%dYF7F554DrX z^@+DS9b;|{mlq^U@ayh7$B`Ql#hwtXin|Qt9k|PA6+e1zEqckFh0XirxI=&Wg1NMx zh-x5#W97i^3UZMxY9Ml=cF;QbxVGtq2;;gXALR!6y{IRgHT`CW_}k3Laq-W772#tS zRhef}_g3VtdBJOth-?oKwb)_BU6dEkpOx#KC}knsJA|UQYn%6?Ya7X=5v%+!Vt4)3 zjnEyhJ?pR@f~4n7gDCyTmyla3XRxHiX_89lSA6Zmh+@f7TF=eqy)ch{oMOqnv!r?6 z?-zoIl1kn%@vZ(v1Ro9tu^o<33ONvrO3nxbPD$h|KlJW5cNT9B`}2rh;B;VJ&Eh~S zjo`AI67{C|NqztLf(Gl(RAp7BsFqnxuc~I!95I`72*BB;q7fa_`b#YlJJl4xdzAL7=N zX2V&a5g&u?#~Y`+1kM^tI`246Hv?zgRxB!eCZ0URvWv8RArYv2sc=Bh8F+H}c9r}E zdjRj3y2q5DNpxoZsz;t>4VG{la5~objpRsJm^>&a#o*@8`=ox?RY?1J_}GTrOues7 zFvxK&*zgxDc$XMho1D6S5+{y1j6Wdw46sqG?|0#zqbXJ?k+b;tn~zi!F>cm zyEBPcIdBSzWP46ZmJ1JMMmxl#@!ZLrtj$e-6P^u$c9=|ky4TJ0pchtLZY$DeOIJqWx$ zq?YBHcYG+18dMhO7WAsygA+I!^o;N$>I^XATd>}60DndhFxrp?fA&RY30_71*?EWd z<`d3}IZI4txv@pjpAjhSV)1y%p(~wRdZCfe`~l7p%T}<@WR%^T*^8ZWl-ilwm}%> z3xVg*XEJSL`G&kjmh)aQxz^{$jT-uJ)y>fp;19oGq04=guRsfbolr<-rldt=KJwaw z9F5{8Ez;EoiB!NG>W-Y#tkbgzl8BVEE5~3`as|mu*nJffO;Apti>1pNz~!orHEP3L z?WiW=*h;@>rbpPea#}`PcBp999vu8r%MhgKbqeWZj8Ok-zNSO>! zAf&YI(aW_yn_u~sYtMqrX=;MAOp~aK7E#79XleqgKvM8vN%)2FQ{|>z@s4JXEXraR z_k+s9VEnB8dW6d29Sg{3C{JQ=fvEa?p)kgUV4t=?iZu~9;SYGi{Vorjlc-3E%r z+Te^cF?cJSLGFe2u)W)B-b+R0mc2u8Px>s&XTNhX1~&WN&=Br*$qAO8j$zmYYQh@+ zp6tgObS8tz&eV}kZR3sd;W+ol5~qEi$k#@Pgj1&1#z+Hg==f~X;xGD+?l%ZtNU!n( z!igjC_hB2yeYb9W_k1kPLKY*SpRLkxPXt3c-~wM|ZLZ>K?ZK$9qPYf{lOg5yeuR#C zaXJs2wXp+iZxCA&nNi@pp}(G#Wu~v+$$E(ER`j3@2&&e`yE@*;y@Zw~c*T3V^WM)T z1S@h<;;a|BP4As;B!Wv3&{Bj18Gfzm2{aL7;dMbCHMA0L>vHyW)mx(sbSUJ|FKn7r zP9Ay7Y}U2U{<(^`qJDq`)!4fFv~i2*oF3$TVG?y5)m?15x#_{wO)R%0X%hVtu>w4e zCGqsN4TdNioVZuD@cFQJ4UFm;6L8%QxM*bUTNl4MC4IYNwaP3ijoNVVO37PgDwKlf z8?T_K_X~MyaAL8;jG0)=x}rlVx;KA7@+f3`{BcBL5N(V`TM^NtdpM5C(h3&z!wRP; zFOCJX@RhvS-hmGgIuMcx=I)WUGW15-CZNbRu{Y7Y!5l`-PyMy#?)}hI!7J{z3;%|t zg$};se5-(f@u?$IA3F2XGlKU!LAg(6$5-Euud(UO+uEPPXB*Dm^XurUI?# zoQcodZSt@+TwWo{ua3Sa(7C4f)rRHWP>Q=+xPhUN{z5}O_mzob^NE&}47BZ_^oj7# zY{~D5jfGYdlY4LwkVI$@5VrrC*pRTaH8pdj_#@2$uyk@Zvo&-4J>BtLZCwLj9OI1& zHl=D+CYzBF6kcrx1x$qI715@r5=yLB6d0nOWQx}y=Q_QLxUlpdyq~Fp7VZ92VfV%> zWYO-1js#8N?zO=0WPbK~*1o=y^7?vium*|~JR5~m$Z=!L=huoyd+)w06kZpEJ1R3O zJ!)++;1>i#BxOpbY5EF}$XC*bey}8Mh#c=tSY6&5gX%@QSqA#nqp{ay4a|u; z#{NaA_-aq%svvaS482R5vpA(o>6qjT{*Y*c4?5MQ^7>ho7sNt_p>WYZrt4f?+X_H z>yJ~XN_$+WBtW`5^WI43a1CF*CwY?$Fs!r|z1^EHI)cIh;;Te?Mc6Q8onwGnkb8EN z$Sb)E7m@ZW)*+Fp1}Wr899e{O5sD)iiWWc40JI^&I!cK<6rb&9FVPx&8;Z&)F~LwI z-Mpd)zAyIRSyM+K;uIKOK@b}gJYPDq99~HJ@Fi+c)fiBrOxoq!#G9Z}RwYfniBFI7 z8zGN>9`V#BtOR!m3A{wjw~o6GD-#mWOe%?SEPCwi~$r#Lzorj)eeX^g%; z=Fm6H=WDSGv>^K;%Bj2J{Vr*X&DQw*)~FXyV@Fd&+rr4j^q41G55n&*Zt}B|JCbII z_URpE~H+up(uEc$m^gQ3h1Wk+D>pem?1UkDaTMIH7B$uZri9S8kM^wqD9- zwEc1o&+Ku)$YVoAbe}Vw(!t5@edzQa!nLjbn(@7NtUE>EfZzPb_3HR}>&cPb`}4FO zNP82ZD0Cr)E?ieIQ2S$cpj2TWQyvMvADCXuh^V1N%pxC76WL%o&tTxskVn4cDu^L^ zcP0DJA_OA3B5-~TG@7dRYF(U0`;?H(^zM`)$0BWby@-&NfmP5dB5h1_+jhO=}1=KtGUGvY^P&+h-eEqEkIXs`?XXAL)B{OxDgPIoRx`u za}}N;-#t>PNKC<>z0yzCuc?qJW=~J3SWMA+x~lNh?@Ak80B9}KC2PT!$1;}P!@3`R za-Ih_Hcj_EiE!3k;M0s;p8c#2F*gq0Z^kHwpfeBA@9Z{Nvo8VAXza_0K~qmxQ*84> zP7Cx{ozv}M69*IIj|wiOf3X>yuJ@wUxIDsoDjlGf+5Wk2C4f?;7;dMOI_ryfU{Xu( z9&R%%lz?(Fu``TG+ZvH6&c#gn#FBPwEd_xhqXo&IN6g*UvZ_d+Cn~a=&P$U6OqIse z3n}ICji*B*q1x=k2HF}u>d>wAi(szzyQz9%ubadIAB2KbUqH@_A95C6Va~6H>DJJZ zh5Px@x^;|8wx~sqiu@R09c5#(HdlucxeDY(oz(Jqzs&uVnOK_3Er!QD-`+=a{j4_- zZQHNe-Q{OPl|EV~RLRTgD>GVSj;7_^HFyBaY@bRab`1z!?h#Q^8=16G*f`l(7<`E7 zJ4`_pr^49EDq<1pkT!YV(~QQ9&{ZGZ?a8a2qOpx`$O04%YzyiM-sZ#?DIb3HJnzcW zYY#?F`nnpLnYX%nOX`rjh5b+v@Cz{6RsHcY-A+H)wO*WtW(Xg%z#Z29)|E9E#GHq>2HCERy(lLlVnH_5(L5?6V?KmA0 zZ*9Y9JJ~urUTI`5*y*Xwwk4qgeSA3D*@nmb;R*&iEQgYQ9Yr$?SOyp6C3ox<7knFyk3pMhKuLTdSeU~33)gOqh5)9i$tv!l#08OO(D?*7Xgwq!);{SE%1DNRtPxelds! zb_G4s_w`X&v)ve84itMbj5f#v_L(K}b-91)R%{-XT%V@6lmMwt;O&sf@A+jTA zR*B(X_idjwPb!VylQgxSLp8NLB}I`&zDl^m-U3bq1H_qPpjHyghT_D}+ zSus}$V1=%$VRdn4c8ta!?{tWsxh#1_-azU19TVjShYX7QWJfEi6!vk9P)erx#81PTXC>LoID3>Z9I;xYwykeL5vBOcF<;iwwt$WX0ZVtV7 zx&gYHCpNI*i@CK9nZ1TJ+vti-OZL#~?_p)5`l{^T5-& z%!jUiX6Aw^7_Vxl<0)RtfZVs<<>&6w*jdM3Y`uL`_UyITgtSZ4&8JjKBJjJx2|R!Q z*FfFhNsV|H6a+-_SFn!%?>nrTqOBqY>+f5ui?W>E^aqUe$P6BwBrZfbS#%j4QE7`V z%L)-)I1wpn;0&|F%0g*xpPppHbQp1`-R259Ds;>NXlPad%2qi#5UelpZtTpnmVdkP z`FbL$5+t=aZO~}8S1>`XxY&NZGu0Ow;nyFJH(S^U+T}jo+2@cIhRNsd$|up|o@4@B zf>~_g8J(MNu*ib-nN+v#8{!Ca__X3p``0vRb-HIfUCpOZbv2S=ymisYJ0dQP;=MxA zeU$G%ktC+fXy)I{J2Gc*kv+;U zNFUHwjWfUq+P=Xvl~Eu0MQ;u=4%1`ZVqvXY158Z3wk4%p)pCf-N}jRR(A{z5!B&RE zb={F}3KNG{g#CQ^yW^50gPjCPI<5|8$f!EKa(1N!@0GlXlU$3L$=KdF(0G>PM6IWO z#QA2XVA3DyxZ~T~Wt^<$B2PopOCqIMl>^WJ}qu7NK*jhHhOt%pNhkBnGlb+m8A3 zoBLnJxF6sTpurHf39_WsraB+3OTOD&z>0c?yoBl75Km@czppHLTzVxHQYep-2UQ`< zUE^zBP&lqkOtTFUT#ufG5!3Wec=({rF!wo-0w^n)VD55=!yeDc-ypV_d&&7j0*E=C z=Da!)GAf^vX+(k6bis`*cr3P4#Wdj0QZGRk52zxC7esMkXtp^R$-TxShtL;XYp_$~ zWFkkc8NgVs88#q&1~8?f*=hi|9%JQP809i{i&-Lf^vV3leO}iyN~KY1^$gH{^A22( ze-bmNGZ)O(=RG19$BdR<^?!TNBz~3JQz#DzWeS*a=$@c>nWDI%=CN-6#zKHXMnJGj zr5zkKEG^$hFEc6s*pU^fyys^W5xMUDC0o!k!Q`RyJv?RuKp1sJDQuQ0{qqs|X3aaO zqX_1dON=P}kbo@x9YyHsl6X784)ull!z;xbxS}j57zX%%_X7QT;(rwskW-Q0>VMdM zes>1_N&U~>px+3;m3&|NU*G?r{*zPae>MJ_1?TtjTjNl__NU(sIe)hK@1~vKEIEHM zX#QaQY0&xayx%N3fAJLmh4+VH=TGu~w(R^#&-eBIck;g&c>e0}f6kHprKFDb&;FwS zyNo}S{#VBA@A`j#dHzd31M7e5|GuXGUGMLq!e4stg#V`Z&w<0=h5o*){3S$4@*jo% VydFrX-=J{6jta0}HN|hg{vRZPS^oe4 literal 0 HcmV?d00001 diff --git a/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom b/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom new file mode 100644 index 00000000..96e0fe50 --- /dev/null +++ b/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom @@ -0,0 +1,9 @@ + + + 4.0.0 + com.dosse + WaifUPnP + 1.1 + POM was created from install:install-file + diff --git a/lib/com/dosse/WaifUPnP/maven-metadata-local.xml b/lib/com/dosse/WaifUPnP/maven-metadata-local.xml new file mode 100644 index 00000000..07d6ffd0 --- /dev/null +++ b/lib/com/dosse/WaifUPnP/maven-metadata-local.xml @@ -0,0 +1,12 @@ + + + com.dosse + WaifUPnP + + 1.1 + + 1.1 + + 20220218200127 + + diff --git a/pom.xml b/pom.xml index 1d7eebeb..f6d68377 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,7 @@ 1.2.2 28.1-jre 2.5.1 + 1.1 2.29.1 9.4.29.v20200521 2.17.1 @@ -427,6 +428,12 @@ AT ${ciyam-at.version} + + + com.dosse + WaifUPnP + ${upnp.version} + org.bitcoinj From 8de606588c22ed815dff06424de80e4e97f0977d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Feb 2022 20:11:00 +0000 Subject: [PATCH 22/54] Attempt to open the listen port (default 12392) using UPnP, if the local network supports it. --- src/main/java/org/qortal/network/Network.java | 6 ++++++ src/main/java/org/qortal/settings/Settings.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index c9ae3b7a..8c1a05aa 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -1,5 +1,6 @@ package org.qortal.network; +import com.dosse.upnp.UPnP; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters; @@ -183,6 +184,11 @@ public class Network { } } + // Attempt to set up UPnP. All errors are ignored. + if (Settings.getInstance().isuPnPEnabled()) { + UPnP.openPortTCP(Settings.getInstance().getListenPort()); + } + // Start up first networking thread networkEPC.start(); } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index a3e03028..d1bc41a8 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -181,6 +181,8 @@ public class Settings { private boolean isTestNet = false; /** Port number for inbound peer-to-peer connections. */ private Integer listenPort; + /** Whether to attempt to open the listen port via UPnP */ + private boolean uPnPEnabled = true; /** Minimum number of peers to allow block minting / synchronization. */ private int minBlockchainPeers = 5; /** Target number of outbound connections to peers we should make. */ @@ -629,6 +631,10 @@ public class Settings { return this.bindAddress; } + public boolean isuPnPEnabled() { + return this.uPnPEnabled; + } + public int getMinBlockchainPeers() { return this.minBlockchainPeers; } From 6d0db7cc5e97ea4a921b1e2d66cf45b5760ccc5b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 19 Feb 2022 17:18:51 +0000 Subject: [PATCH 23/54] Catch UncheckedIOException in findAllHostedPaths() which was seen when a file was deleted by another thread. --- .../controller/arbitrary/ArbitraryDataStorageManager.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index 16aac458..e2649cbe 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -16,6 +16,7 @@ import org.qortal.utils.FilesystemUtils; import org.qortal.utils.NTP; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -322,7 +323,7 @@ public class ArbitraryDataStorageManager extends Thread { && path.getFileName().toString().length() > 32) .collect(Collectors.toList()); } - catch (IOException e) { + catch (IOException | UncheckedIOException e) { LOGGER.info("Unable to walk through hosted data: {}", e.getMessage()); } From 5d419dd4ec864426b70bd88a28c805f4ee234498 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 19 Feb 2022 17:45:24 +0000 Subject: [PATCH 24/54] Handle case where funds are sent to and from the same bitcoiny deterministic wallet. --- src/main/java/org/qortal/crosschain/Bitcoiny.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index cce20c84..bc6f79e1 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -476,6 +476,7 @@ public abstract class Bitcoiny implements ForeignBlockchain { List outputs = new ArrayList<>(); boolean anyOutputAddressInWallet = false; + boolean transactionInvolvesExternalWallet = false; for (BitcoinyTransaction.Input input : t.inputs) { try { @@ -490,6 +491,9 @@ public abstract class Bitcoiny implements ForeignBlockchain { total += inputAmount; addressInWallet = true; } + else { + transactionInvolvesExternalWallet = true; + } inputs.add(new SimpleTransaction.Input(sender, inputAmount, addressInWallet)); } } @@ -503,14 +507,17 @@ public abstract class Bitcoiny implements ForeignBlockchain { for (String address : output.addresses) { boolean addressInWallet = false; if (keySet.contains(address)) { - if (total > 0L) { + if (total > 0L) { // Change returned from sent amount amount -= (total - output.value); - } else { + } else { // Amount received amount += output.value; } addressInWallet = true; anyOutputAddressInWallet = true; } + else { + transactionInvolvesExternalWallet = true; + } outputs.add(new SimpleTransaction.Output(address, output.value, addressInWallet)); } } @@ -525,6 +532,10 @@ public abstract class Bitcoiny implements ForeignBlockchain { amount = total * -1; } } + else if (!transactionInvolvesExternalWallet) { + // All inputs and outputs relate to this wallet, so the balance should be unaffected + amount = 0; + } return new SimpleTransaction(t.txHash, t.timestamp, amount, fee, inputs, outputs); } From 7307844beefe458bba432b94a1d3e8571497dd5f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 20 Feb 2022 10:44:20 +0000 Subject: [PATCH 25/54] If UPnP is disabled in settings, close the existing external listenPort if a UPnP rule exists. --- src/main/java/org/qortal/network/Network.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 8c1a05aa..8811ef6c 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -188,6 +188,9 @@ public class Network { if (Settings.getInstance().isuPnPEnabled()) { UPnP.openPortTCP(Settings.getInstance().getListenPort()); } + else { + UPnP.closePortTCP(Settings.getInstance().getListenPort()); + } // Start up first networking thread networkEPC.start(); From 21b4b494e7b3e13c73ddfdaa01d6e4562de9eebf Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 20 Feb 2022 10:44:38 +0000 Subject: [PATCH 26/54] Renamed method. --- src/main/java/org/qortal/network/Network.java | 2 +- src/main/java/org/qortal/settings/Settings.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 8811ef6c..304e27ef 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -185,7 +185,7 @@ public class Network { } // Attempt to set up UPnP. All errors are ignored. - if (Settings.getInstance().isuPnPEnabled()) { + if (Settings.getInstance().isUPnPEnabled()) { UPnP.openPortTCP(Settings.getInstance().getListenPort()); } else { diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index d1bc41a8..779c29f5 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -631,7 +631,7 @@ public class Settings { return this.bindAddress; } - public boolean isuPnPEnabled() { + public boolean isUPnPEnabled() { return this.uPnPEnabled; } From f4f7cc58e3f9a5361fc1ccd12ef8d5202ff66636 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 20 Feb 2022 10:44:59 +0000 Subject: [PATCH 27/54] Removed unused import. --- src/main/java/org/qortal/network/Network.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 304e27ef..d3502b03 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -8,7 +8,6 @@ import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters; import org.qortal.block.BlockChain; import org.qortal.controller.Controller; import org.qortal.controller.arbitrary.ArbitraryDataFileListManager; -import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.crypto.Crypto; import org.qortal.data.block.BlockData; import org.qortal.data.network.PeerData; From 146e7970bfb1e9bf0ea6c510d36ec2976e336912 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 20 Feb 2022 11:04:33 +0000 Subject: [PATCH 28/54] Synchronize this.allKnownPeers and this.connectedPeers in Network.requestDataFromPeer(), to make the method thread-safe. This could be further improved by taking an immutable copy, but I'll leave this until the same approach is applied to other Network methods. --- src/main/java/org/qortal/network/Network.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index d3502b03..6a5177ad 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -251,12 +251,15 @@ public class Network { public boolean requestDataFromPeer(String peerAddressString, byte[] signature) { if (peerAddressString != null) { PeerAddress peerAddress = PeerAddress.fromString(peerAddressString); + PeerData peerData = null; // Reuse an existing PeerData instance if it's already in the known peers list - PeerData peerData = this.allKnownPeers.stream() - .filter(knownPeerData -> knownPeerData.getAddress().equals(peerAddress)) - .findFirst() - .orElse(null); + synchronized (this.allKnownPeers) { + peerData = this.allKnownPeers.stream() + .filter(knownPeerData -> knownPeerData.getAddress().equals(peerAddress)) + .findFirst() + .orElse(null); + } if (peerData == null) { // Not a known peer, so we need to create one @@ -271,10 +274,13 @@ public class Network { } // Check if we're already connected to and handshaked with this peer - Peer connectedPeer = this.connectedPeers.stream() - .filter(p -> p.getPeerData().getAddress().equals(peerAddress)) - .findFirst() - .orElse(null); + Peer connectedPeer = null; + synchronized (this.connectedPeers) { + connectedPeer = this.connectedPeers.stream() + .filter(p -> p.getPeerData().getAddress().equals(peerAddress)) + .findFirst() + .orElse(null); + } boolean isConnected = (connectedPeer != null); boolean isHandshaked = this.getHandshakedPeers().stream() From 7798b8dcdcd0f90f5b8d9263036b276333a99705 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 20 Feb 2022 11:33:09 +0000 Subject: [PATCH 29/54] Keep items in arbitraryDataFileHashResponses if they are currently being requested by another thread. This should help to locate the higher numbered chunks from larger resources. --- .../controller/arbitrary/ArbitraryDataFileManager.java | 4 ++-- .../arbitrary/ArbitraryDataFileRequestThread.java | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 6d352b79..9b437e2b 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -37,7 +37,7 @@ public class ArbitraryDataFileManager extends Thread { /** * Map to keep track of our in progress (outgoing) arbitrary data file requests */ - private Map arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>()); + public Map arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>()); /** * Map to keep track of hashes that we might need to relay @@ -148,7 +148,7 @@ public class ArbitraryDataFileManager extends Thread { } } else { - LOGGER.trace("Already requesting data file {} for signature {}", arbitraryDataFile, Base58.encode(signature)); + LOGGER.trace("Already requesting data file {} for signature {} from peer {}", arbitraryDataFile, Base58.encode(signature), peer); } } else { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java index 0c2834d0..e46fd2fb 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -80,6 +80,12 @@ public class ArbitraryDataFileRequestThread implements Runnable { continue; } + // Skip if already requesting, but don't remove, as we might want to retry later + if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) { + // Already requesting - leave this attempt for later + continue; + } + // We want to process this file shouldProcess = true; iterator.remove(); From e07adbd60ea8a42f9561420cac7af2171eaac955 Mon Sep 17 00:00:00 2001 From: proto <34919827+protoniuman@users.noreply.github.com> Date: Mon, 21 Feb 2022 15:40:10 +0100 Subject: [PATCH 30/54] online accounts api call, fix level zero accounts Added online zero level accounts to the response of /addresses/online/levels api endpoints --- src/main/java/org/qortal/account/Account.java | 23 ++++++++++++++++++- .../api/resource/AddressesResource.java | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/account/Account.java b/src/main/java/org/qortal/account/Account.java index 417dde6d..aeff7810 100644 --- a/src/main/java/org/qortal/account/Account.java +++ b/src/main/java/org/qortal/account/Account.java @@ -272,7 +272,7 @@ public class Account { /** * Returns 'effective' minting level, or zero if reward-share does not exist. *

- * For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config. + * this is being used on src/main/java/org/qortal/api/resource/AddressesResource.java to fulfil the online accounts api call * * @param repository * @param rewardSharePublicKey @@ -288,5 +288,26 @@ public class Account { Account rewardShareMinter = new Account(repository, rewardShareData.getMinter()); return rewardShareMinter.getEffectiveMintingLevel(); } + /** + * Returns 'effective' minting level, with a fix for the zero level. + *

+ * For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config. + * + * @param repository + * @param rewardSharePublicKey + * @return 0+ + * @throws DataException + */ + public static int getRewardShareEffectiveMintingLevelIncludingLevelZero(Repository repository, byte[] rewardSharePublicKey) throws DataException { + // Find actual minter and get their effective minting level + RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey); + if (rewardShareData == null) + return 0; + else if(!rewardShareData.getMinter().equals(rewardShareData.getRecipient()))//the minter is different than the recipient this means sponsorship + return 0; + + Account rewardShareMinter = new Account(repository, rewardShareData.getMinter()); + return rewardShareMinter.getEffectiveMintingLevel(); + } } diff --git a/src/main/java/org/qortal/api/resource/AddressesResource.java b/src/main/java/org/qortal/api/resource/AddressesResource.java index abe1960c..39c76cf4 100644 --- a/src/main/java/org/qortal/api/resource/AddressesResource.java +++ b/src/main/java/org/qortal/api/resource/AddressesResource.java @@ -198,7 +198,7 @@ public class AddressesResource { for (OnlineAccountData onlineAccountData : onlineAccounts) { try { - final int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, onlineAccountData.getPublicKey()); + final int minterLevel = Account.getRewardShareEffectiveMintingLevelIncludingLevelZero(repository, onlineAccountData.getPublicKey()); OnlineAccountLevel onlineAccountLevel = onlineAccountLevels.stream() .filter(a -> a.getLevel() == minterLevel) From 6b83927048c144d904d2f1301a608e0647e03825 Mon Sep 17 00:00:00 2001 From: proto <34919827+protoniuman@users.noreply.github.com> Date: Mon, 21 Feb 2022 16:17:17 +0100 Subject: [PATCH 31/54] Persist MintingAccounts.json on minting accounts add/remove this fix the behavior of the node, After adding or removing a minting account, allowing it to persist it to the backup folder --- src/main/java/org/qortal/api/resource/AdminResource.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index bde4bed4..7f4a3806 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -315,6 +315,7 @@ public class AdminResource { repository.getAccountRepository().save(mintingAccountData); repository.saveChanges(); + repository.exportNodeLocalData();//after adding new minting account let's persist it to the backup MintingAccounts.json } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e); } catch (DataException e) { @@ -355,6 +356,7 @@ public class AdminResource { return "false"; repository.saveChanges(); + repository.exportNodeLocalData();//after removing new minting account let's persist it to the backup MintingAccounts.json } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e); } catch (DataException e) { @@ -546,7 +548,7 @@ public class AdminResource { @Path("/repository/data") @Operation( summary = "Export sensitive/node-local data from repository.", - description = "Exports data to .script files on local machine" + description = "Exports data to .json files on local machine" ) @ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE}) @SecurityRequirement(name = "apiKey") From 8d7be7757f58a10441e569a38e801a0f7b68fa60 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 21 Feb 2022 22:27:44 +0000 Subject: [PATCH 32/54] Fixed incorrectly named tag. --- .../org/qortal/api/domainmap/resource/DomainMapResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java b/src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java index 27770449..cc21587d 100644 --- a/src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java +++ b/src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java @@ -17,7 +17,7 @@ import java.util.Map; @Path("/") -@Tag(name = "Gateway") +@Tag(name = "Domain Map") public class DomainMapResource { @Context HttpServletRequest request; From 8673c7ef6e0abefa55288a317aa0191f7f202e5d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 21 Feb 2022 22:28:18 +0000 Subject: [PATCH 33/54] Fixed bug in GET /peers/summary API --- src/main/java/org/qortal/api/resource/PeersResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/PeersResource.java b/src/main/java/org/qortal/api/resource/PeersResource.java index 97e2644e..38461141 100644 --- a/src/main/java/org/qortal/api/resource/PeersResource.java +++ b/src/main/java/org/qortal/api/resource/PeersResource.java @@ -354,7 +354,7 @@ public class PeersResource { List connectedPeers = Network.getInstance().getConnectedPeers().stream().collect(Collectors.toList()); for (Peer peer : connectedPeers) { - if (peer.isOutbound()) { + if (!peer.isOutbound()) { peersSummary.inboundConnections++; } else { From a2c462b3dabbb92d815053d3ca1962b3fc628ed9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 21 Feb 2022 22:28:59 +0000 Subject: [PATCH 34/54] Add tag to websites. Fixed issue rendering emojis and other special characters. --- src/main/java/org/qortal/api/HTMLParser.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java index 474b6417..026d9210 100644 --- a/src/main/java/org/qortal/api/HTMLParser.java +++ b/src/main/java/org/qortal/api/HTMLParser.java @@ -28,6 +28,11 @@ public class HTMLParser { // Add base href tag String baseElement = String.format("", baseUrl); head.get(0).prepend(baseElement); + + // Add meta charset tag + String metaCharsetElement = ""; + head.get(0).prepend(metaCharsetElement); + } String html = document.html(); this.data = html.getBytes(); From d5c3921846371b4ae8650fd75754a16714cdbae3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 21 Feb 2022 22:34:13 +0000 Subject: [PATCH 35/54] Only show the red "synchronizing" systray icon if the latest block isn't recent. This should fix issue where the icon unnecessarily jumps between synced and synchronizing. --- src/main/java/org/qortal/controller/Controller.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 2bf7d973..fb1cbd47 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -767,14 +767,14 @@ public class Controller extends Thread { actionText = Translator.INSTANCE.translate("SysTray", "MINTING_ENABLED"); SysTray.getInstance().setTrayIcon(2); } - else if (Synchronizer.getInstance().isSynchronizing()) { - actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), Synchronizer.getInstance().getSyncPercent()); - SysTray.getInstance().setTrayIcon(3); - } else if (numberOfPeers < Settings.getInstance().getMinBlockchainPeers()) { actionText = Translator.INSTANCE.translate("SysTray", "CONNECTING"); SysTray.getInstance().setTrayIcon(3); } + else if (!this.isUpToDate()) { + actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), Synchronizer.getInstance().getSyncPercent()); + SysTray.getInstance().setTrayIcon(3); + } else { actionText = Translator.INSTANCE.translate("SysTray", "MINTING_DISABLED"); SysTray.getInstance().setTrayIcon(4); From a3753c01bc4c3c73f02630496e5fe18dba1be156 Mon Sep 17 00:00:00 2001 From: proto <34919827+protoniuman@users.noreply.github.com> Date: Tue, 22 Feb 2022 15:50:46 +0100 Subject: [PATCH 36/54] Add search functionality to hosted resources --- .../api/resource/ArbitraryResource.java | 16 ++++- .../ArbitraryDataStorageManager.java | 58 +++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 8031bf83..f67423dd 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -432,14 +432,24 @@ public class ArbitraryResource { @HeaderParam(Security.API_KEY_HEADER) String apiKey, @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus, @Parameter(ref = "limit") @QueryParam("limit") Integer limit, - @Parameter(ref = "offset") @QueryParam("offset") Integer offset) { + @Parameter(ref = "offset") @QueryParam("offset") Integer offset, + @Parameter(ref = "query") @QueryParam("query") String query) { + Security.checkApiCallAllowed(request); List resources = new ArrayList<>(); try (final Repository repository = RepositoryManager.getRepository()) { + + List transactionDataList; + + if(query.equals("")){ + transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset); + }else{ + transactionDataList = ArbitraryDataStorageManager.getInstance().searchHostedTransactions(repository,query, limit, offset); + } + - List transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset); for (ArbitraryTransactionData transactionData : transactionDataList) { ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); arbitraryResourceInfo.name = transactionData.getName(); @@ -461,6 +471,8 @@ public class ArbitraryResource { } } + + @DELETE @Path("/resource/{service}/{name}/{identifier}") @Operation( diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index e2649cbe..75e37692 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -47,6 +47,9 @@ public class ArbitraryDataStorageManager extends Thread { private List hostedTransactions; + private String searchQuery; + private List searchResultsTransactions; + private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes /** Treat storage as full at 90% usage, to reduce risk of going over the limit. @@ -301,6 +304,61 @@ public class ArbitraryDataStorageManager extends Thread { return ArbitraryTransactionUtils.limitOffsetTransactions(arbitraryTransactionDataList, limit, offset); } + + /** + * searchHostedTransactions + * Allow to run a query against hosted data names and return matches if there are any + * @param repository + * @param query + * @param limit + * @param offset + * @return + */ + + public List searchHostedTransactions(Repository repository, String query, Integer limit, Integer offset) { + // Load from cache if we can (results that exists for the same query), to avoid disk reads + if (this.searchResultsTransactions != null && this.searchQuery.equals(query.toLowerCase())) { + return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset); + } + this.searchQuery=query.toLowerCase();//set the searchQuery so that it can be checked on the next call + + List searchResultsList = new ArrayList<>(); + + // Find all hosted paths + List allPaths = this.findAllHostedPaths(); + + // Loop through each path and attempt to match it to a signature, and check if it's a match with our search query + for (Path path : allPaths) { + try { + String[] contents = path.toFile().list(); + if (contents == null || contents.length == 0) { + // Ignore empty directories + continue; + } + + String signature58 = path.getFileName().toString(); + byte[] signature = Base58.decode(signature58); + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (transactionData == null || transactionData.getType() != Transaction.TransactionType.ARBITRARY) { + continue; + } + ArbitraryTransactionData arbitraryTransactionData =(ArbitraryTransactionData) transactionData; + if(arbitraryTransactionData.getName().toLowerCase().contains(this.searchQuery) || arbitraryTransactionData.getIdentifier().toLowerCase().contains(this.searchQuery)) + searchResultsList.add(arbitraryTransactionData); + + } catch (DataException e) { + continue; + } + } + + // Sort by newest first + searchResultsList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed()); + + // Update cache + this.searchResultsTransactions = searchResultsList; + + return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset); + } /** * Warning: this method will walk through the entire data directory From 782904a9711268448533e5174051d441b34c07ed Mon Sep 17 00:00:00 2001 From: proto <34919827+protoniuman@users.noreply.github.com> Date: Tue, 22 Feb 2022 17:54:08 +0100 Subject: [PATCH 37/54] improvement to the search on hosted resources 1) use the cached version instead of rescanning all the files 2) separating the loading (which include files scanning) and listing logic --- .../api/resource/ArbitraryResource.java | 2 +- .../ArbitraryDataStorageManager.java | 62 +++++++++---------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index f67423dd..d6eb71ff 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -443,7 +443,7 @@ public class ArbitraryResource { List transactionDataList; - if(query.equals("")){ + if(query==null || query.equals("")){ transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset); }else{ transactionDataList = ArbitraryDataStorageManager.getInstance().searchHostedTransactions(repository,query, limit, offset); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index 75e37692..fc97fdf8 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -261,14 +261,8 @@ public class ArbitraryDataStorageManager extends Thread { } - // Hosted data - - public List listAllHostedTransactions(Repository repository, Integer limit, Integer offset) { - // Load from cache if we can, to avoid disk reads - if (this.hostedTransactions != null) { - return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset); - } - + public List loadAllHostedTransactions(Repository repository){ + List arbitraryTransactionDataList = new ArrayList<>(); // Find all hosted paths @@ -299,10 +293,21 @@ public class ArbitraryDataStorageManager extends Thread { // Sort by newest first arbitraryTransactionDataList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed()); - // Update cache - this.hostedTransactions = arbitraryTransactionDataList; - return ArbitraryTransactionUtils.limitOffsetTransactions(arbitraryTransactionDataList, limit, offset); + return arbitraryTransactionDataList; + } + // Hosted data + + public List listAllHostedTransactions(Repository repository, Integer limit, Integer offset) { + // Load from cache if we can, to avoid disk reads + + if (this.hostedTransactions != null) { + return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset); + } + + this.hostedTransactions = this.loadAllHostedTransactions(repository); + + return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset); } /** @@ -316,37 +321,28 @@ public class ArbitraryDataStorageManager extends Thread { */ public List searchHostedTransactions(Repository repository, String query, Integer limit, Integer offset) { - // Load from cache if we can (results that exists for the same query), to avoid disk reads + // Load from results cache if we can (results that exists for the same query), to avoid disk reads if (this.searchResultsTransactions != null && this.searchQuery.equals(query.toLowerCase())) { return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset); } + + // Using cache if we can, to avoid disk reads + if (this.hostedTransactions == null) { + this.hostedTransactions = this.loadAllHostedTransactions(repository); + } + this.searchQuery=query.toLowerCase();//set the searchQuery so that it can be checked on the next call List searchResultsList = new ArrayList<>(); - // Find all hosted paths - List allPaths = this.findAllHostedPaths(); - - // Loop through each path and attempt to match it to a signature, and check if it's a match with our search query - for (Path path : allPaths) { + // Loop through cached hostedTransactions + for (ArbitraryTransactionData atd : this.hostedTransactions) { try { - String[] contents = path.toFile().list(); - if (contents == null || contents.length == 0) { - // Ignore empty directories - continue; - } + + if(atd.getName().toLowerCase().contains(this.searchQuery) || atd.getIdentifier().toLowerCase().contains(this.searchQuery)) + searchResultsList.add(atd); - String signature58 = path.getFileName().toString(); - byte[] signature = Base58.decode(signature58); - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (transactionData == null || transactionData.getType() != Transaction.TransactionType.ARBITRARY) { - continue; - } - ArbitraryTransactionData arbitraryTransactionData =(ArbitraryTransactionData) transactionData; - if(arbitraryTransactionData.getName().toLowerCase().contains(this.searchQuery) || arbitraryTransactionData.getIdentifier().toLowerCase().contains(this.searchQuery)) - searchResultsList.add(arbitraryTransactionData); - - } catch (DataException e) { + } catch (Exception e) { continue; } } From e339ab856fd4fb845664cdeb69f5d909b8378ec2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 23 Feb 2022 08:34:38 +0000 Subject: [PATCH 38/54] Skip over Electrum servers that don't return any output addresses. Hopeful fix for BTC transactions that report a zero value due to incomplete data being returned from certain ElectrumX peers. --- src/main/java/org/qortal/crosschain/ElectrumX.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index 7f1eb4c4..d26d963f 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -410,6 +410,16 @@ public class ElectrumX extends BitcoinyBlockchainProvider { addresses.add((String) addressObj); } + // For the purposes of Qortal we require all outputs to contain addresses + // Some servers omit this info, causing problems down the line with balance calculations + if (addresses.isEmpty()) { + if (this.currentServer != null) { + this.uselessServers.add(this.currentServer); + this.closeServer(this.currentServer); + } + throw new ForeignBlockchainException(String.format("No output addresses returned for transaction %s", txHash)); + } + outputs.add(new BitcoinyTransaction.Output(scriptPubKey, value, addresses)); } From 2d493a4ea2c905040ab7d9bcae6d61427d67afe2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 23 Feb 2022 09:29:16 +0000 Subject: [PATCH 39/54] Added logging when no addresses are returned for a bitcoiny transaction output. --- src/main/java/org/qortal/crosschain/ElectrumX.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index d26d963f..d6e35efc 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -417,6 +417,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider { this.uselessServers.add(this.currentServer); this.closeServer(this.currentServer); } + LOGGER.info("No output addresses returned for transaction {}", txHash); throw new ForeignBlockchainException(String.format("No output addresses returned for transaction %s", txHash)); } From 0271ef69c9160ad10cc6c3606abb8c725412650c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 23 Feb 2022 20:06:55 +0000 Subject: [PATCH 40/54] When submitting a new transaction, treat the chain as "synced" if the latest block is less than 30 minutes old. Increased from around 7.5 minutes. --- .../api/resource/TransactionsResource.java | 5 ++++- .../org/qortal/controller/Controller.java | 19 ++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/TransactionsResource.java b/src/main/java/org/qortal/api/resource/TransactionsResource.java index 9bc6d497..55ad7cde 100644 --- a/src/main/java/org/qortal/api/resource/TransactionsResource.java +++ b/src/main/java/org/qortal/api/resource/TransactionsResource.java @@ -638,7 +638,10 @@ public class TransactionsResource { ApiError.BLOCKCHAIN_NEEDS_SYNC, ApiError.INVALID_SIGNATURE, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE }) public String processTransaction(String rawBytes58) { - if (!Controller.getInstance().isUpToDate()) + // Only allow a transaction to be processed if our latest block is less than 30 minutes old + // If older than this, we should first wait until the blockchain is synced + final Long minLatestBlockTimestamp = NTP.getTime() - (30 * 60 * 1000L); + if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC); byte[] rawBytes = Base58.decode(rawBytes58); diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index fb1cbd47..542a2889 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -2048,10 +2048,13 @@ public class Controller extends Thread { return peers; } - /** Returns whether we think our node has up-to-date blockchain based on our info about other peers. */ - public boolean isUpToDate() { + /** + * Returns whether we think our node has up-to-date blockchain based on our info about other peers. + * @param minLatestBlockTimestamp - the minimum block timestamp to be considered recent + * @return boolean - whether our node's blockchain is up to date or not + */ + public boolean isUpToDate(Long minLatestBlockTimestamp) { // Do we even have a vaguely recent block? - final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp(); if (minLatestBlockTimestamp == null) return false; @@ -2077,6 +2080,16 @@ public class Controller extends Thread { return !peers.isEmpty(); } + /** + * Returns whether we think our node has up-to-date blockchain based on our info about other peers. + * Uses the default minLatestBlockTimestamp value. + * @return boolean - whether our node's blockchain is up to date or not + */ + public boolean isUpToDate() { + final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp(); + return this.isUpToDate(minLatestBlockTimestamp); + } + /** Returns minimum block timestamp for block to be considered 'recent', or null if NTP not synced. */ public static Long getMinimumLatestBlockTimestamp() { Long now = NTP.getTime(); From 9e6fe7ceb936c634467e2d054b0bc864795b5ae5 Mon Sep 17 00:00:00 2001 From: catbref Date: Thu, 24 Feb 2022 09:06:21 +0000 Subject: [PATCH 41/54] Modify TradeBot, some related websockets, to trigger when chain tip changes instead of with every new block --- .../api/websocket/PresenceWebSocket.java | 7 +++--- .../api/websocket/TradeOffersWebSocket.java | 5 +++-- .../org/qortal/controller/Synchronizer.java | 22 +++++++++++++++++++ .../qortal/controller/tradebot/TradeBot.java | 3 ++- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java b/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java index 26d131c4..c579ac86 100644 --- a/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java @@ -20,6 +20,7 @@ import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; import org.eclipse.jetty.websocket.api.annotations.WebSocket; import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.crypto.Crypto; import org.qortal.data.transaction.PresenceTransactionData; import org.qortal.data.transaction.TransactionData; @@ -99,13 +100,13 @@ public class PresenceWebSocket extends ApiWebSocket implements Listener { @Override public void listen(Event event) { - // We use NewBlockEvent as a proxy for 1-minute timer - if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Controller.NewBlockEvent)) + // We use Synchronizer.NewChainTipEvent as a proxy for 1-minute timer + if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Synchronizer.NewChainTipEvent)) return; removeOldEntries(); - if (event instanceof Controller.NewBlockEvent) + if (event instanceof Synchronizer.NewChainTipEvent) // We only wanted a chance to cull old entries return; diff --git a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java index 186f79e3..35fc4691 100644 --- a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java @@ -23,6 +23,7 @@ import org.eclipse.jetty.websocket.api.annotations.WebSocket; import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; import org.qortal.api.model.CrossChainOfferSummary; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.crosschain.SupportedBlockchain; import org.qortal.crosschain.ACCT; import org.qortal.crosschain.AcctMode; @@ -80,10 +81,10 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener { @Override public void listen(Event event) { - if (!(event instanceof Controller.NewBlockEvent)) + if (!(event instanceof Synchronizer.NewChainTipEvent)) return; - BlockData blockData = ((Controller.NewBlockEvent) event).getBlockData(); + BlockData blockData = ((Synchronizer.NewChainTipEvent) event).getNewChainTip(); // Process any new info diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index b98c5fa2..bb36d42d 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -22,6 +22,8 @@ import org.qortal.data.block.CommonBlockData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.transaction.RewardShareTransactionData; import org.qortal.data.transaction.TransactionData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; import org.qortal.network.Network; import org.qortal.network.Peer; import org.qortal.network.message.BlockMessage; @@ -96,6 +98,24 @@ public class Synchronizer extends Thread { OK, NOTHING_TO_DO, GENESIS_ONLY, NO_COMMON_BLOCK, TOO_DIVERGENT, NO_REPLY, INFERIOR_CHAIN, INVALID_DATA, NO_BLOCKCHAIN_LOCK, REPOSITORY_ISSUE, SHUTTING_DOWN; } + public static class NewChainTipEvent implements Event { + private final BlockData priorChainTip; + private final BlockData newChainTip; + + public NewChainTipEvent(BlockData priorChainTip, BlockData newChainTip) { + this.priorChainTip = priorChainTip; + this.newChainTip = newChainTip; + } + + public BlockData getPriorChainTip() { + return this.priorChainTip; + } + + public BlockData getNewChainTip() { + return this.newChainTip; + } + } + // Constructors private Synchronizer() { @@ -338,6 +358,8 @@ public class Synchronizer extends Thread { Network network = Network.getInstance(); network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip)); + + EventBus.INSTANCE.notify(new NewChainTipEvent(priorChainTip, newChainTip)); } return syncResult; diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index 2f9c3121..6996acbb 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -16,6 +16,7 @@ import org.bitcoinj.core.ECKey; import org.qortal.account.PrivateKeyAccount; import org.qortal.api.model.crosschain.TradeBotCreateRequest; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult; import org.qortal.crosschain.*; import org.qortal.data.at.ATData; @@ -213,7 +214,7 @@ public class TradeBot implements Listener { @Override public void listen(Event event) { - if (!(event instanceof Controller.NewBlockEvent)) + if (!(event instanceof Synchronizer.NewChainTipEvent)) return; synchronized (this) { From 8950bb7af940f20ece39b8df5f6f2a47857e8d20 Mon Sep 17 00:00:00 2001 From: catbref Date: Thu, 24 Feb 2022 09:13:51 +0000 Subject: [PATCH 42/54] Very slightly relax validity checks for TRANSFER_PRIVS to allow for skeletal account records, e.g. due to CHAT transactions, but account last reference still needs to be null. Example at block height 736196 / 7 --- .../java/org/qortal/transaction/TransferPrivsTransaction.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/transaction/TransferPrivsTransaction.java b/src/main/java/org/qortal/transaction/TransferPrivsTransaction.java index f77dac15..f6a9de68 100644 --- a/src/main/java/org/qortal/transaction/TransferPrivsTransaction.java +++ b/src/main/java/org/qortal/transaction/TransferPrivsTransaction.java @@ -58,7 +58,9 @@ public class TransferPrivsTransaction extends Transaction { return ValidationResult.INVALID_ADDRESS; // Check recipient is new account - if (this.repository.getAccountRepository().accountExists(this.transferPrivsTransactionData.getRecipient())) + AccountData recipientAccountData = this.repository.getAccountRepository().getAccount(this.transferPrivsTransactionData.getRecipient()); + // Non-existent account data is OK, but if account data exists then reference needs to be null + if (recipientAccountData != null && recipientAccountData.getReference() != null) return ValidationResult.ACCOUNT_ALREADY_EXISTS; // Check sender has funds for fee From a63ef4010d5c911d9fbfcf207e024f023fe33b06 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 24 Feb 2022 19:05:29 +0000 Subject: [PATCH 43/54] Disabled expired transaction data deletion code for now, as this was often causing data to be incorrectly deleted. This will need to be re-enabled at some point, but only after it's modified to be much less aggressive. --- .../controller/arbitrary/ArbitraryDataCleanupManager.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index 64916df5..3edf0ef8 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -222,7 +222,11 @@ public class ArbitraryDataCleanupManager extends Thread { try (final Repository repository = RepositoryManager.getRepository()) { // Check if there are any hosted files that don't have matching transactions - this.checkForExpiredTransactions(repository); + // UPDATE: This has been disabled for now as it was deleting valid transactions + // and causing chunks to go missing on the network. If ever re-enabled, we MUST + // ensure that original copies of data aren't deleted, and that sufficient time + // is allowed (ideally several hours) before treating a transaction as missing. + // this.checkForExpiredTransactions(repository); // Delete additional data at random if we're over our storage limit // Use the DELETION_THRESHOLD so that we only start deleting once the hard limit is reached From d5521068b0fdcedf1dc3035db11c86c1b2dc9177 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 24 Feb 2022 19:45:37 +0000 Subject: [PATCH 44/54] Fixed issue in earlier commit, found in unit tests. --- src/main/java/org/qortal/crosschain/ElectrumX.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index d6e35efc..b6fe15f9 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -412,7 +412,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider { // For the purposes of Qortal we require all outputs to contain addresses // Some servers omit this info, causing problems down the line with balance calculations - if (addresses.isEmpty()) { + if (addresses == null || addresses.isEmpty()) { if (this.currentServer != null) { this.uselessServers.add(this.currentServer); this.closeServer(this.currentServer); From 53c4fe9e801692d84fecc3484b49dd9b7d85ae8b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 24 Feb 2022 20:01:56 +0000 Subject: [PATCH 45/54] Fixed another ElectrumX issue found in unit tests. Peers that were thought to be missing output address data may actually have just been using a different key - "address" instead of "addresses". Now reading the addresses from both keys, which may remove the need for the previously added checks. --- .../java/org/qortal/crosschain/ElectrumX.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index b6fe15f9..6d6cfb15 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -401,17 +401,29 @@ public class ElectrumX extends BitcoinyBlockchainProvider { String scriptPubKey = (String) ((JSONObject) outputJson.get("scriptPubKey")).get("hex"); long value = BigDecimal.valueOf((Double) outputJson.get("value")).setScale(8).unscaledValue().longValue(); - // address too, if present + // address too, if present in the "addresses" array List addresses = null; Object addressesObj = ((JSONObject) outputJson.get("scriptPubKey")).get("addresses"); if (addressesObj instanceof JSONArray) { addresses = new ArrayList<>(); - for (Object addressObj : (JSONArray) addressesObj) + for (Object addressObj : (JSONArray) addressesObj) { addresses.add((String) addressObj); + } + } + + // some peers return a single "address" string + Object addressObj = ((JSONObject) outputJson.get("scriptPubKey")).get("address"); + if (addressObj instanceof String) { + if (addresses == null) { + addresses = new ArrayList<>(); + } + addresses.add((String) addressObj); } // For the purposes of Qortal we require all outputs to contain addresses // Some servers omit this info, causing problems down the line with balance calculations + // Update: it turns out that they were just using a different key - "address" instead of "addresses" + // The code below can remain in place, just in case a peer returns a missing address in the future if (addresses == null || addresses.isEmpty()) { if (this.currentServer != null) { this.uselessServers.add(this.currentServer); From 31d34c3946c1e241b76ec3677780fb76d296994f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 25 Feb 2022 11:08:37 +0000 Subject: [PATCH 46/54] Updated testnet documentation --- TestNets.md | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/TestNets.md b/TestNets.md index 6f8e92e6..e475e593 100644 --- a/TestNets.md +++ b/TestNets.md @@ -41,13 +41,39 @@ - Start up at least as many nodes as `minBlockchainPeers` (or adjust this value instead) - Probably best to perform API call `DELETE /peers/known` - Add other nodes via API call `POST /peers ` -- Add minting private key to node(s) via API call `POST /admin/mintingaccounts ` - This key must have corresponding `REWARD_SHARE` transaction in testnet genesis block +- Add minting private key to nodes via API call `POST /admin/mintingaccounts ` + The keys must have corresponding `REWARD_SHARE` transactions in testnet genesis block +- You must have at least 2 separate minting keys and two separate nodes. Assign one minting key to each node. +- Alternatively, comment out the `if (mintedLastBlock) { }` conditional in BlockMinter.java to allow for a single node and key. - Wait for genesis block timestamp to pass - A node should mint block 2 approximately 60 seconds after genesis block timestamp - Other testnet nodes will sync *as long as there is at least `minBlockchainPeers` peers with an "up-to-date" chain` - You can also use API call `POST /admin/forcesync ` on stuck nodes +## Single-node testnet + +A single-node testnet is possible with code modifications, for basic testing, or to more easily start a new testnet. +To do so, follow these steps: +- Comment out the `if (mintedLastBlock) { }` conditional in BlockMinter.java +- Comment out the `minBlockchainPeers` validation in Settings.validate() +- Set `minBlockchainPeers` to 0 in settings.json +- Set `Synchronizer.RECOVERY_MODE_TIMEOUT` to `0` +- All other steps should remain the same. Only a single reward share key is needed. +- Remember to put these values back after introducing other nodes + +## Fixed network + +To restrict a testnet to a set of private nodes, you can use the "fixed network" feature. +This ensures that the testnet nodes only communicate with each other and not other known peers. +To do this, add the following setting to each testnet node, substituting the IP addresses: +``` +"fixedNetwork": [ + "192.168.0.101:62392", + "192.168.0.102:62392", + "192.168.0.103:62392" +] +``` + ## Dealing with stuck chain Maybe your nodes have been offline and no-one has minted a recent testnet block. From 1a5937916226c65d981ed08dbeb53dbc937a5a3a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Feb 2022 13:25:20 +0000 Subject: [PATCH 47/54] Optionally include requestTime, requestHops, peerAddress, and isRelayPossible flag in ArbitraryDataFileListMessage --- .../ArbitraryDataFileListManager.java | 14 +++- .../data/arbitrary/ArbitraryRelayInfo.java | 14 +++- src/main/java/org/qortal/network/Network.java | 5 ++ .../message/ArbitraryDataFileListMessage.java | 83 +++++++++++++++++-- 4 files changed, 106 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index 87620128..a521922c 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -472,14 +472,22 @@ public class ArbitraryDataFileListManager { if (!isBlocked) { Peer requestingPeer = request.getB(); if (requestingPeer != null) { + Long requestTime = arbitraryDataFileListMessage.getRequestTime(); + Integer requestHops = arbitraryDataFileListMessage.getRequestHops(); + // Add each hash to our local mapping so we know who to ask later Long now = NTP.getTime(); for (byte[] hash : hashes) { String hash58 = Base58.encode(hash); - ArbitraryRelayInfo relayMap = new ArbitraryRelayInfo(hash58, signature58, peer, now); + ArbitraryRelayInfo relayMap = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops); ArbitraryDataFileManager.getInstance().addToRelayMap(relayMap); } + // Bump requestHops if it exists + if (requestHops != null) { + arbitraryDataFileListMessage.setRequestHops(++requestHops); + } + // Forward to requesting peer LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer); if (!requestingPeer.sendMessage(arbitraryDataFileListMessage)) { @@ -582,7 +590,9 @@ public class ArbitraryDataFileListManager { arbitraryDataFileListRequests.put(message.getId(), newEntry); } - ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + String ourAddress = Network.getInstance().getOurExternalIpAddress(); + ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, + hashes, NTP.getTime(), 0, ourAddress, true); arbitraryDataFileListMessage.setId(message.getId()); if (!peer.sendMessage(arbitraryDataFileListMessage)) { LOGGER.debug("Couldn't send list of hashes"); diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java index 94f41d18..17c1acac 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java @@ -9,12 +9,16 @@ public class ArbitraryRelayInfo { private final String signature58; private final Peer peer; private final Long timestamp; + private final Long requestTime; + private final Integer requestHops; - public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp) { + public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp, Long requestTime, Integer requestHops) { this.hash58 = hash58; this.signature58 = signature58; this.peer = peer; this.timestamp = timestamp; + this.requestTime = requestTime; + this.requestHops = requestHops; } public boolean isValid() { @@ -38,6 +42,14 @@ public class ArbitraryRelayInfo { return timestamp; } + public Long getRequestTime() { + return this.requestTime; + } + + public Integer getRequestHops() { + return this.requestHops; + } + @Override public String toString() { return String.format("%s = %s, %s, %d", this.hash58, this.signature58, this.peer, this.timestamp); diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 6a5177ad..5de0b84d 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -1195,6 +1195,11 @@ public class Network { //ArbitraryDataManager.getInstance().broadcastHostedSignatureList(); } + public String getOurExternalIpAddress() { + // FUTURE: replace port if UPnP is active, as it will be more accurate + return this.ourExternalIpAddress; + } + // Peer-management calls diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java index 008b3edd..3eef284f 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java @@ -1,6 +1,8 @@ package org.qortal.network.message; import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.qortal.data.network.PeerData; import org.qortal.transform.TransformationException; import org.qortal.transform.Transformer; import org.qortal.utils.Serialization; @@ -16,22 +18,38 @@ public class ArbitraryDataFileListMessage extends Message { private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; private static final int HASH_LENGTH = Transformer.SHA256_LENGTH; + private static final int MAX_PEER_ADDRESS_LENGTH = PeerData.MAX_PEER_ADDRESS_SIZE; private final byte[] signature; private final List hashes; + private final Long requestTime; + private Integer requestHops; + private final String peerAddress; + private final boolean isRelayPossible; - public ArbitraryDataFileListMessage(byte[] signature, List hashes) { + + public ArbitraryDataFileListMessage(byte[] signature, List hashes, Long requestTime, + Integer requestHops, String peerAddress, boolean isRelayPossible) { super(MessageType.ARBITRARY_DATA_FILE_LIST); this.signature = signature; this.hashes = hashes; + this.requestTime = requestTime; + this.requestHops = requestHops; + this.peerAddress = peerAddress; + this.isRelayPossible = isRelayPossible; } - public ArbitraryDataFileListMessage(int id, byte[] signature, List hashes) { + public ArbitraryDataFileListMessage(int id, byte[] signature, List hashes, Long requestTime, + Integer requestHops, String peerAddress, boolean isRelayPossible) { super(id, MessageType.ARBITRARY_DATA_FILE_LIST); this.signature = signature; this.hashes = hashes; + this.requestTime = requestTime; + this.requestHops = requestHops; + this.peerAddress = peerAddress; + this.isRelayPossible = isRelayPossible; } public List getHashes() { @@ -48,9 +66,6 @@ public class ArbitraryDataFileListMessage extends Message { int count = bytes.getInt(); - if (bytes.remaining() != count * HASH_LENGTH) - return null; - List hashes = new ArrayList<>(); for (int i = 0; i < count; ++i) { @@ -59,7 +74,26 @@ public class ArbitraryDataFileListMessage extends Message { hashes.add(hash); } - return new ArbitraryDataFileListMessage(id, signature, hashes); + Long requestTime = null; + Integer requestHops = null; + String peerAddress = null; + boolean isRelayPossible = true; // Legacy versions only send this message when relaying is possible + + // The remaining fields are optional + + if (bytes.hasRemaining()) { + + requestTime = bytes.getLong(); + + requestHops = bytes.getInt(); + + peerAddress = Serialization.deserializeSizedStringV2(bytes, MAX_PEER_ADDRESS_LENGTH); + + isRelayPossible = bytes.getInt() > 0; + + } + + return new ArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops, peerAddress, isRelayPossible); } @Override @@ -75,6 +109,20 @@ public class ArbitraryDataFileListMessage extends Message { bytes.write(hash); } + if (this.requestTime == null) { // Just in case + return bytes.toByteArray(); + } + + // The remaining fields are optional + + bytes.write(Longs.toByteArray(this.requestTime)); + + bytes.write(Ints.toByteArray(this.requestHops)); + + Serialization.serializeSizedStringV2(bytes, this.peerAddress); + + bytes.write(Ints.toByteArray(this.isRelayPossible ? 1 : 0)); + return bytes.toByteArray(); } catch (IOException e) { return null; @@ -82,9 +130,30 @@ public class ArbitraryDataFileListMessage extends Message { } public ArbitraryDataFileListMessage cloneWithNewId(int newId) { - ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes); + ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes, + this.requestTime, this.requestHops, this.peerAddress, this.isRelayPossible); clone.setId(newId); return clone; } + public Long getRequestTime() { + return this.requestTime; + } + + public Integer getRequestHops() { + return this.requestHops; + } + + public void setRequestHops(int requestHops) { + this.requestHops = requestHops; + } + + public String getPeerAddress() { + return this.peerAddress; + } + + public boolean isRelayPossible() { + return this.isRelayPossible; + } + } From 1864468818955edeada607b0be0cc0fdcdbcd649 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Feb 2022 13:41:29 +0000 Subject: [PATCH 48/54] Prefer the route with the least number of hops when relaying. --- .../arbitrary/ArbitraryDataFileManager.java | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 9b437e2b..e60e32a9 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -392,6 +392,33 @@ public class ArbitraryDataFileManager extends Thread { } } + private ArbitraryRelayInfo getOptimalRelayInfoEntryForHash(String hash58) { + LOGGER.trace("Fetching relay info for hash: {}", hash58); + List relayInfoList = this.getRelayInfoListForHash(hash58); + if (relayInfoList != null && !relayInfoList.isEmpty()) { + + // Remove any with null requestHops + relayInfoList.removeIf(r -> r.getRequestHops() == null); + + // If list is now empty, then just return one at random + if (relayInfoList.isEmpty()) { + return this.getRandomRelayInfoEntryForHash(hash58); + } + + // Sort by number of hops (lowest first) + relayInfoList.sort(Comparator.comparingInt(ArbitraryRelayInfo::getRequestHops)); + + // FUTURE: secondary sort by requestTime? + + ArbitraryRelayInfo relayInfo = relayInfoList.get(0); + + LOGGER.trace("Returning optimal relay info for hash: {} (requestHops {})", hash58, relayInfo.getRequestHops()); + return relayInfo; + } + LOGGER.trace("No relay info exists for hash: {}", hash58); + return null; + } + private ArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) { LOGGER.trace("Fetching random relay info for hash: {}", hash58); List relayInfoList = this.getRelayInfoListForHash(hash58); @@ -442,7 +469,7 @@ public class ArbitraryDataFileManager extends Thread { try { ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); - ArbitraryRelayInfo relayInfo = this.getRandomRelayInfoEntryForHash(hash58); + ArbitraryRelayInfo relayInfo = this.getOptimalRelayInfoEntryForHash(hash58); if (arbitraryDataFile.exists()) { LOGGER.trace("Hash {} exists", hash58); From 3dadce4da426507df83908da6dc7a6dbf60b2c3b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Feb 2022 14:17:27 +0000 Subject: [PATCH 49/54] Renamed a reference --- .../controller/arbitrary/ArbitraryDataFileListManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index a521922c..21e1c106 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -479,8 +479,8 @@ public class ArbitraryDataFileListManager { Long now = NTP.getTime(); for (byte[] hash : hashes) { String hash58 = Base58.encode(hash); - ArbitraryRelayInfo relayMap = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops); - ArbitraryDataFileManager.getInstance().addToRelayMap(relayMap); + ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops); + ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo); } // Bump requestHops if it exists From eac4b0d87b462a8d93dc2336de31b50ea47a1ef8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 20 Feb 2022 20:11:53 +0000 Subject: [PATCH 50/54] Maintain backwards support for pre-3.2.0 peers by only including new file list message params when sending to newer peers. These params are optional and the process will function without them, just less efficiently. --- .../ArbitraryDataFileListManager.java | 14 +++++++++ .../message/ArbitraryDataFileListMessage.java | 31 +++++++++++++++---- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index 21e1c106..a10dd847 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -29,6 +29,7 @@ public class ArbitraryDataFileListManager { private static ArbitraryDataFileListManager instance; + private static String MIN_PEER_VERSION_FOR_FILE_LIST_STATS = "3.2.0"; /** * Map of recent incoming requests for ARBITRARY transaction data file lists. @@ -488,6 +489,12 @@ public class ArbitraryDataFileListManager { arbitraryDataFileListMessage.setRequestHops(++requestHops); } + // Remove optional parameters if the requesting peer doesn't support it yet + // A message with less statistical data is better than no message at all + if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { + arbitraryDataFileListMessage.removeOptionalStats(); + } + // Forward to requesting peer LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer); if (!requestingPeer.sendMessage(arbitraryDataFileListMessage)) { @@ -594,6 +601,13 @@ public class ArbitraryDataFileListManager { ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, NTP.getTime(), 0, ourAddress, true); arbitraryDataFileListMessage.setId(message.getId()); + + // Remove optional parameters if the requesting peer doesn't support it yet + // A message with less statistical data is better than no message at all + if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { + arbitraryDataFileListMessage.removeOptionalStats(); + } + if (!peer.sendMessage(arbitraryDataFileListMessage)) { LOGGER.debug("Couldn't send list of hashes"); peer.disconnect("failed to send list of hashes"); diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java index 3eef284f..32ba3fa7 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java @@ -22,10 +22,10 @@ public class ArbitraryDataFileListMessage extends Message { private final byte[] signature; private final List hashes; - private final Long requestTime; + private Long requestTime; private Integer requestHops; - private final String peerAddress; - private final boolean isRelayPossible; + private String peerAddress; + private Boolean isRelayPossible; public ArbitraryDataFileListMessage(byte[] signature, List hashes, Long requestTime, @@ -109,7 +109,7 @@ public class ArbitraryDataFileListMessage extends Message { bytes.write(hash); } - if (this.requestTime == null) { // Just in case + if (this.requestTime == null) { // To maintain backwards support return bytes.toByteArray(); } @@ -136,15 +136,26 @@ public class ArbitraryDataFileListMessage extends Message { return clone; } + public void removeOptionalStats() { + this.requestTime = null; + this.requestHops = null; + this.peerAddress = null; + this.isRelayPossible = null; + } + public Long getRequestTime() { return this.requestTime; } + public void setRequestTime(Long requestTime) { + this.requestTime = requestTime; + } + public Integer getRequestHops() { return this.requestHops; } - public void setRequestHops(int requestHops) { + public void setRequestHops(Integer requestHops) { this.requestHops = requestHops; } @@ -152,8 +163,16 @@ public class ArbitraryDataFileListMessage extends Message { return this.peerAddress; } - public boolean isRelayPossible() { + public void setPeerAddress(String peerAddress) { + this.peerAddress = peerAddress; + } + + public Boolean isRelayPossible() { return this.isRelayPossible; } + public void setIsRelayPossible(Boolean isRelayPossible) { + this.isRelayPossible = isRelayPossible; + } + } From 0111747016996508fb820e769ffd6a3963863e4b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 25 Feb 2022 13:30:07 +0000 Subject: [PATCH 51/54] Added debug logging of new file list stats. --- .../controller/arbitrary/ArbitraryDataFileListManager.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index a10dd847..a1ceaa4f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -404,6 +404,13 @@ public class ArbitraryDataFileListManager { ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size()); + if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) { + long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime(); + LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}", + totalRequestTime, arbitraryDataFileListMessage.getRequestHops(), + arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); + } + // Do we have a pending request for this data? Triple request = arbitraryDataFileListRequests.get(message.getId()); if (request == null || request.getA() == null) { From c65a63fc7ef940e0e5f5e9ca6933f6593d16d4c4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 26 Feb 2022 13:59:53 +0000 Subject: [PATCH 52/54] Fixed "query" parameter error in swagger documentation --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index d6eb71ff..5c07fdd1 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -433,7 +433,7 @@ public class ArbitraryResource { @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus, @Parameter(ref = "limit") @QueryParam("limit") Integer limit, @Parameter(ref = "offset") @QueryParam("offset") Integer offset, - @Parameter(ref = "query") @QueryParam("query") String query) { + @QueryParam("query") String query) { Security.checkApiCallAllowed(request); From badb57699142c79f7812e70b53aedc993cfaec3a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 26 Feb 2022 14:04:35 +0000 Subject: [PATCH 53/54] Fixed exception when identifier is null. Also handling null names as this may be a future scenario. --- .../arbitrary/ArbitraryDataStorageManager.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index fc97fdf8..63b70691 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -338,9 +338,12 @@ public class ArbitraryDataStorageManager extends Thread { // Loop through cached hostedTransactions for (ArbitraryTransactionData atd : this.hostedTransactions) { try { - - if(atd.getName().toLowerCase().contains(this.searchQuery) || atd.getIdentifier().toLowerCase().contains(this.searchQuery)) - searchResultsList.add(atd); + if (atd.getName() != null && atd.getName().toLowerCase().contains(this.searchQuery)) { + searchResultsList.add(atd); + } + else if (atd.getIdentifier() != null && atd.getIdentifier().toLowerCase().contains(this.searchQuery)) { + searchResultsList.add(atd); + } } catch (Exception e) { continue; From 7740f3da7eade7903dd0179eb81ee2001ad90d21 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 26 Feb 2022 14:05:28 +0000 Subject: [PATCH 54/54] Small formatting tweaks, for consistency with existing code. --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 5 ++--- .../controller/arbitrary/ArbitraryDataStorageManager.java | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 5c07fdd1..ba39825c 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -443,13 +443,12 @@ public class ArbitraryResource { List transactionDataList; - if(query==null || query.equals("")){ + if (query == null || query.equals("")) { transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset); - }else{ + } else { transactionDataList = ArbitraryDataStorageManager.getInstance().searchHostedTransactions(repository,query, limit, offset); } - for (ArbitraryTransactionData transactionData : transactionDataList) { ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); arbitraryResourceInfo.name = transactionData.getName(); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index 63b70691..4b975b40 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -293,7 +293,6 @@ public class ArbitraryDataStorageManager extends Thread { // Sort by newest first arbitraryTransactionDataList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed()); - return arbitraryTransactionDataList; } // Hosted data @@ -331,7 +330,7 @@ public class ArbitraryDataStorageManager extends Thread { this.hostedTransactions = this.loadAllHostedTransactions(repository); } - this.searchQuery=query.toLowerCase();//set the searchQuery so that it can be checked on the next call + this.searchQuery = query.toLowerCase(); //set the searchQuery so that it can be checked on the next call List searchResultsList = new ArrayList<>();