From 8e8c0b3fc5c55f19044a395e2f26e22ca86ebc16 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 1 Jul 2022 22:29:05 +0100 Subject: [PATCH 01/83] Added OnlineAccountsV3Message, along with optional nonce Integer in OnlineAccountData. This could potentially be released ahead of the other mempow code, splitting the rollout into multiple smaller phases. --- .../org/qortal/controller/Controller.java | 4 + .../controller/OnlineAccountsManager.java | 31 ++++- .../data/network/OnlineAccountData.java | 12 +- .../qortal/network/message/MessageType.java | 2 +- .../message/OnlineAccountsV3Message.java | 121 ++++++++++++++++++ 5 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index cde965c1..b333bd34 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1245,6 +1245,10 @@ public class Controller extends Thread { OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsV3Message(peer, message); break; + case ONLINE_ACCOUNTS_V3: + OnlineAccountsManager.getInstance().onNetworkOnlineAccountsV3Message(peer, message); + break; + case GET_ARBITRARY_DATA: // Not currently supported break; diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 529cc853..c1f58b43 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -445,10 +445,10 @@ public class OnlineAccountsManager { Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts); Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts); - Message messageV3 = new OnlineAccountsV2Message(ourOnlineAccounts); // TODO: V3 message + Message messageV3 = new OnlineAccountsV3Message(ourOnlineAccounts); Network.getInstance().broadcast(peer -> - peer.getPeersVersion() >= ONLINE_ACCOUNTS_V3_PEER_VERSION + peer.getPeersVersion() >= OnlineAccountsV3Message.MIN_PEER_VERSION ? messageV3 : peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 @@ -715,9 +715,32 @@ public class OnlineAccountsManager { } } - Message onlineAccountsMessage = new OnlineAccountsV2Message(outgoingOnlineAccounts); // TODO: V3 message - peer.sendMessage(onlineAccountsMessage); + peer.sendMessage( + peer.getPeersVersion() >= OnlineAccountsV3Message.MIN_PEER_VERSION ? + new OnlineAccountsV3Message(outgoingOnlineAccounts) : + new OnlineAccountsV2Message(outgoingOnlineAccounts) + ); LOGGER.debug("Sent {} online accounts to {}", outgoingOnlineAccounts.size(), peer); } + + public void onNetworkOnlineAccountsV3Message(Peer peer, Message message) { + OnlineAccountsV3Message onlineAccountsMessage = (OnlineAccountsV3Message) message; + + List peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts(); + LOGGER.debug("Received {} online accounts from {}", peersOnlineAccounts.size(), peer); + + int importCount = 0; + + // Add any online accounts to the queue that aren't already present + for (OnlineAccountData onlineAccountData : peersOnlineAccounts) { + boolean isNewEntry = onlineAccountsImportQueue.add(onlineAccountData); + + if (isNewEntry) + importCount++; + } + + if (importCount > 0) + LOGGER.debug("Added {} online accounts to queue", importCount); + } } diff --git a/src/main/java/org/qortal/data/network/OnlineAccountData.java b/src/main/java/org/qortal/data/network/OnlineAccountData.java index 28c454b5..bd4842db 100644 --- a/src/main/java/org/qortal/data/network/OnlineAccountData.java +++ b/src/main/java/org/qortal/data/network/OnlineAccountData.java @@ -16,6 +16,7 @@ public class OnlineAccountData { protected long timestamp; protected byte[] signature; protected byte[] publicKey; + protected Integer nonce; @XmlTransient private int hash; @@ -26,10 +27,15 @@ public class OnlineAccountData { protected OnlineAccountData() { } - public OnlineAccountData(long timestamp, byte[] signature, byte[] publicKey) { + public OnlineAccountData(long timestamp, byte[] signature, byte[] publicKey, Integer nonce) { this.timestamp = timestamp; this.signature = signature; this.publicKey = publicKey; + this.nonce = nonce; + } + + public OnlineAccountData(long timestamp, byte[] signature, byte[] publicKey) { + this(timestamp, signature, publicKey, null); } public long getTimestamp() { @@ -44,6 +50,10 @@ public class OnlineAccountData { return this.publicKey; } + public Integer getNonce() { + return this.nonce; + } + // For JAXB @XmlElement(name = "address") protected String getAddress() { diff --git a/src/main/java/org/qortal/network/message/MessageType.java b/src/main/java/org/qortal/network/message/MessageType.java index de711dc3..087e7fbf 100644 --- a/src/main/java/org/qortal/network/message/MessageType.java +++ b/src/main/java/org/qortal/network/message/MessageType.java @@ -46,7 +46,7 @@ public enum MessageType { GET_ONLINE_ACCOUNTS(81, GetOnlineAccountsMessage::fromByteBuffer), ONLINE_ACCOUNTS_V2(82, OnlineAccountsV2Message::fromByteBuffer), GET_ONLINE_ACCOUNTS_V2(83, GetOnlineAccountsV2Message::fromByteBuffer), - // ONLINE_ACCOUNTS_V3(84, OnlineAccountsV3Message::fromByteBuffer), + ONLINE_ACCOUNTS_V3(84, OnlineAccountsV3Message::fromByteBuffer), GET_ONLINE_ACCOUNTS_V3(85, GetOnlineAccountsV3Message::fromByteBuffer), ARBITRARY_DATA(90, ArbitraryDataMessage::fromByteBuffer), diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java new file mode 100644 index 00000000..cdd52939 --- /dev/null +++ b/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java @@ -0,0 +1,121 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.qortal.data.network.OnlineAccountData; +import org.qortal.transform.Transformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * For sending online accounts info to remote peer. + * + * Same format as V2, but with added support for a mempow nonce. + */ +public class OnlineAccountsV3Message extends Message { + + public static final long MIN_PEER_VERSION = 0x300050000L; // 3.5.0 + + private List onlineAccounts; + + public OnlineAccountsV3Message(List onlineAccounts) { + super(MessageType.ONLINE_ACCOUNTS_V3); + + // Shortcut in case we have no online accounts + if (onlineAccounts.isEmpty()) { + this.dataBytes = Ints.toByteArray(0); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + return; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + Long timestamp = onlineAccountData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + try { + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + if (onlineAccountData.getTimestamp() == timestamp) { + bytes.write(onlineAccountData.getSignature()); + bytes.write(onlineAccountData.getPublicKey()); + + // Nonce is optional; use -1 as placeholder if missing + int nonce = onlineAccountData.getNonce() != null ? onlineAccountData.getNonce() : -1; + bytes.write(Ints.toByteArray(nonce)); + } + } + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private OnlineAccountsV3Message(int id, List onlineAccounts) { + super(id, MessageType.ONLINE_ACCOUNTS_V3); + + this.onlineAccounts = onlineAccounts; + } + + public List getOnlineAccounts() { + return this.onlineAccounts; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { + int accountCount = bytes.getInt(); + + List onlineAccounts = new ArrayList<>(accountCount); + + while (accountCount > 0) { + long timestamp = bytes.getLong(); + + for (int i = 0; i < accountCount; ++i) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; + bytes.get(signature); + + byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + bytes.get(publicKey); + + // Nonce is optional - will be -1 if missing + Integer nonce = bytes.getInt(); + if (nonce < 0) { + nonce = null; + } + + onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey, nonce)); + } + + if (bytes.hasRemaining()) { + accountCount = bytes.getInt(); + } else { + // we've finished + accountCount = 0; + } + } + + return new OnlineAccountsV3Message(id, onlineAccounts); + } + +} From d2adadb6002ab952dfa7ecafda4347ea01f8d23b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 1 Jul 2022 22:31:30 +0100 Subject: [PATCH 02/83] Added onlineAccountsMemoryPoWTimestamp to blockchain.json --- src/main/java/org/qortal/block/BlockChain.java | 8 ++++++++ src/main/resources/blockchain.json | 1 + 2 files changed, 9 insertions(+) diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 8bbefb11..c79d3a30 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -169,6 +169,10 @@ public class BlockChain { * featureTriggers because unit tests need to set this value via Reflection. */ private long onlineAccountsModulusV2Timestamp; + /** Feature trigger timestamp for online accounts mempow verification. Can't use featureTriggers + * because unit tests need to set this value via Reflection. */ + private long onlineAccountsMemoryPoWTimestamp; + /** Settings relating to CIYAM AT feature. */ public static class CiyamAtSettings { /** Fee per step/op-code executed. */ @@ -322,6 +326,10 @@ public class BlockChain { return this.onlineAccountsModulusV2Timestamp; } + public long getOnlineAccountsMemoryPoWTimestamp() { + return this.onlineAccountsMemoryPoWTimestamp; + } + /** Returns true if approval-needing transaction types require a txGroupId other than NO_GROUP. */ public boolean getRequireGroupForApproval() { return this.requireGroupForApproval; diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index dc84ebd0..d4a9124d 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -20,6 +20,7 @@ "onlineAccountSignaturesMinLifetime": 43200000, "onlineAccountSignaturesMaxLifetime": 86400000, "onlineAccountsModulusV2Timestamp": 9999999999999, + "onlineAccountsMemoryPoWTimestamp": 9999999999999, "rewardsByHeight": [ { "height": 1, "reward": 5.00 }, { "height": 259201, "reward": 4.75 }, From b05d428b2e32afaaa9328f0ad2b5a66727f112f3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 1 Jul 2022 22:31:51 +0100 Subject: [PATCH 03/83] Added onlineAccountsMemPoWEnabled setting (for beta testing) --- src/main/java/org/qortal/settings/Settings.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index e0ed7306..ed4d3cd6 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -283,6 +283,11 @@ public class Settings { /** Additional offset added to values returned by NTP.getTime() */ private Long testNtpOffset = null; + // Online accounts + + /** Whether to opt-in to mempow computations for online accounts, ahead of general release */ + private boolean onlineAccountsMemPoWEnabled = false; + // Data storage (QDN) @@ -766,6 +771,10 @@ public class Settings { return this.testNtpOffset; } + public boolean isOnlineAccountsMemPoWEnabled() { + return this.onlineAccountsMemPoWEnabled; + } + public long getRepositoryBackupInterval() { return this.repositoryBackupInterval; } From 215800fb67ca05e8c3d7df8ad4e3b5a0b7384660 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 1 Jul 2022 22:36:51 +0100 Subject: [PATCH 04/83] Added optional "timeout" parameter to MemoryPoW.compute2(). This can be used to give up after the specified number of milliseconds. --- .../java/org/qortal/crypto/MemoryPoW.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/main/java/org/qortal/crypto/MemoryPoW.java b/src/main/java/org/qortal/crypto/MemoryPoW.java index 01f4f6fd..f27c8f7a 100644 --- a/src/main/java/org/qortal/crypto/MemoryPoW.java +++ b/src/main/java/org/qortal/crypto/MemoryPoW.java @@ -1,10 +1,44 @@ package org.qortal.crypto; +import org.qortal.utils.NTP; + import java.nio.ByteBuffer; +import java.util.concurrent.TimeoutException; public class MemoryPoW { + /** + * Compute a MemoryPoW nonce + * + * @param data + * @param workBufferLength + * @param difficulty + * @return + * @throws TimeoutException + */ public static Integer compute2(byte[] data, int workBufferLength, long difficulty) { + try { + return MemoryPoW.compute2(data, workBufferLength, difficulty, null); + + } catch (TimeoutException e) { + // This won't happen, because above timeout is null + return null; + } + } + + /** + * Compute a MemoryPoW nonce, with optional timeout + * + * @param data + * @param workBufferLength + * @param difficulty + * @param timeout maximum number of milliseconds to compute for before giving up,
or null if no timeout + * @return + * @throws TimeoutException + */ + public static Integer compute2(byte[] data, int workBufferLength, long difficulty, Long timeout) throws TimeoutException { + long startTime = NTP.getTime(); + // Hash data with SHA256 byte[] hash = Crypto.digest(data); @@ -33,6 +67,13 @@ public class MemoryPoW { if (Thread.currentThread().isInterrupted()) return -1; + if (timeout != null) { + long now = NTP.getTime(); + if (now > startTime + timeout) { + throw new TimeoutException("Timeout reached"); + } + } + seed *= seedMultiplier; // per nonce state[0] = longHash[0] ^ seed; From 294582f1360393eb9f766a137121f04b1b7950d9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 2 Jul 2022 12:33:02 +0100 Subject: [PATCH 05/83] Added mempow support in OnlineAccountsManager. - This adds mempow requirements to online account importing (after activation timestamp), however doesn't yet add any requirements to block validation. - It also causes the 'next' online accounts timestamp to be computed in addition to the 'current', so that the computed nonce value is ready when the next online accounts timestamp window begins. --- .../controller/OnlineAccountsManager.java | 200 +++++++++++++++++- 1 file changed, 194 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index c1f58b43..a919ed2a 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -9,6 +9,7 @@ import org.qortal.account.PrivateKeyAccount; import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; +import org.qortal.crypto.MemoryPoW; import org.qortal.crypto.Qortal25519Extras; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; @@ -19,10 +20,13 @@ import org.qortal.network.message.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; import org.qortal.utils.Base58; import org.qortal.utils.NTP; import org.qortal.utils.NamedThreadFactory; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.util.*; import java.util.concurrent.*; import java.util.stream.Collectors; @@ -52,11 +56,15 @@ public class OnlineAccountsManager { private static final long ONLINE_ACCOUNTS_QUEUE_INTERVAL = 100L; //ms private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms private static final long ONLINE_ACCOUNTS_LEGACY_BROADCAST_INTERVAL = 60 * 1000L; // ms - private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 15 * 1000L; // ms + private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 5 * 1000L; // ms private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L; // v3.2.0 private static final long ONLINE_ACCOUNTS_V3_PEER_VERSION = 0x0300040000L; // v3.4.0 + // MemoryPoW + public final int POW_BUFFER_SIZE = 1 * 1024 * 1024; // bytes + public int POW_DIFFICULTY = 18; // leading zero bits + private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(4, new NamedThreadFactory("OnlineAccounts")); private volatile boolean isStopping = false; @@ -95,6 +103,10 @@ public class OnlineAccountsManager { return (now / onlineTimestampModulus) * onlineTimestampModulus; } + public static long toOnlineAccountTimestamp(long timestamp) { + return (timestamp / getOnlineTimestampModulus()) * getOnlineTimestampModulus(); + } + private OnlineAccountsManager() { } @@ -190,6 +202,52 @@ public class OnlineAccountsManager { } } + /** + * Check if supplied onlineAccountData is superior (i.e. has a nonce value) than existing record. + * Two entries are considered equal even if the nonce differs, to prevent multiple variations + * co-existing. For this reason, we need to be able to check if a new OnlineAccountData entry should + * replace the existing one, which may be missing the nonce. + * @param onlineAccountData + * @return true if supplied data is superior to existing entry + */ + private boolean isOnlineAccountsDataSuperior(OnlineAccountData onlineAccountData) { + if (onlineAccountData.getNonce() == null || onlineAccountData.getNonce() < 0) { + // New online account data has no usable nonce value, so it won't be better than anything we already have + return false; + } + + // New online account data has a nonce value, so check if there is any existing data to compare against + Set existingOnlineAccountsForTimestamp = this.currentOnlineAccounts.get(onlineAccountData.getTimestamp()); + if (existingOnlineAccountsForTimestamp == null) { + // No existing online accounts data with this timestamp yet + return false; + } + + // Check if a duplicate entry exists + OnlineAccountData existingOnlineAccountData = null; + for (OnlineAccountData existingAccount : existingOnlineAccountsForTimestamp) { + if (existingAccount.equals(onlineAccountData)) { + // Found existing online account data + existingOnlineAccountData = existingAccount; + break; + } + } + + if (existingOnlineAccountData == null) { + // No existing online accounts data, so nothing to compare + return false; + } + + if (existingOnlineAccountData.getNonce() == null || existingOnlineAccountData.getNonce() < 0) { + // Existing data has no usable nonce value(s) so we want to replace it with the new one + return true; + } + + // Both new and old data have nonce values so the new data isn't considered superior + return false; + } + + // Utilities public static byte[] xorByteArrayInPlace(byte[] inplaceArray, byte[] otherArray) { @@ -242,6 +300,14 @@ public class OnlineAccountsManager { return false; } + // Validate mempow if feature trigger is active + if (now >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + if (!getInstance().verifyMemoryPoW(onlineAccountData)) { + LOGGER.trace(() -> String.format("Rejecting online reward-share for account %s due to invalid PoW nonce", mintingAccount.getAddress())); + return false; + } + } + return true; } @@ -299,6 +365,12 @@ public class OnlineAccountsManager { long onlineAccountTimestamp = onlineAccountData.getTimestamp(); Set onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountTimestamp, k -> ConcurrentHashMap.newKeySet()); + + boolean isSuperiorEntry = isOnlineAccountsDataSuperior(onlineAccountData); + if (isSuperiorEntry) + // Remove existing inferior entry so it can be re-added below (it's likely the existing copy is missing a nonce value) + onlineAccounts.remove(onlineAccountData); + boolean isNewEntry = onlineAccounts.add(onlineAccountData); if (isNewEntry) @@ -386,13 +458,26 @@ public class OnlineAccountsManager { return; } + // 'next' timestamp (prioritize this as it's the most important) + final long nextOnlineAccountsTimestamp = toOnlineAccountTimestamp(now) + getOnlineTimestampModulus(); + boolean success = computeOurAccountsForTimestamp(nextOnlineAccountsTimestamp); + if (!success) { + // We didn't compute the required nonce value(s), and so can't proceed until they have been retried + return; + } + + // 'current' timestamp + computeOurAccountsForTimestamp(onlineAccountsTimestamp); + } + + private boolean computeOurAccountsForTimestamp(long onlineAccountsTimestamp) { List mintingAccounts; try (final Repository repository = RepositoryManager.getRepository()) { mintingAccounts = repository.getAccountRepository().getMintingAccounts(); // We have no accounts to send if (mintingAccounts.isEmpty()) - return; + return false; // Only active reward-shares allowed Iterator iterator = mintingAccounts.iterator(); @@ -415,7 +500,7 @@ public class OnlineAccountsManager { } } catch (DataException e) { LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage())); - return; + return false; } final boolean useAggregateCompatibleSignature = onlineAccountsTimestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp(); @@ -427,13 +512,46 @@ public class OnlineAccountsManager { byte[] privateKey = mintingAccountData.getPrivateKey(); byte[] publicKey = Crypto.toPublicKey(privateKey); + // Generate bytes for mempow + byte[] mempowBytes; + try { + mempowBytes = this.getMemoryPoWBytes(publicKey, onlineAccountsTimestamp); + } + catch (IOException e) { + LOGGER.info("Unable to create bytes for MemoryPoW. Moving on to next account..."); + continue; + } + + // Compute nonce + Integer nonce; + if (isMemoryPoWActive()) { + try { + nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp); + if (nonce == null) { + // A nonce is required + return false; + } + } catch (TimeoutException e) { + LOGGER.info(String.format("Timed out computing nonce for account %.8s", Base58.encode(publicKey))); + return false; + } + } + else { + // Send -1 if we haven't computed a nonce due to feature trigger timestamp + nonce = -1; + } + byte[] signature = useAggregateCompatibleSignature ? Qortal25519Extras.signForAggregation(privateKey, timestampBytes) : Crypto.sign(privateKey, timestampBytes); // Our account is online - OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey); - ourOnlineAccounts.add(ourOnlineAccountData); + OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce); + + // Make sure to verify before adding + if (verifyMemoryPoW(ourOnlineAccountData)) { + ourOnlineAccounts.add(ourOnlineAccountData); + } } this.hasOurOnlineAccounts = !ourOnlineAccounts.isEmpty(); @@ -441,7 +559,7 @@ public class OnlineAccountsManager { boolean hasInfoChanged = addAccounts(ourOnlineAccounts); if (!hasInfoChanged) - return; + return false; Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts); Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts); @@ -456,8 +574,78 @@ public class OnlineAccountsManager { ); LOGGER.debug("Broadcasted {} online account{} with timestamp {}", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp); + + return true; } + + + // MemoryPoW + + private boolean isMemoryPoWActive() { + Long now = NTP.getTime(); + if (now < BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp() || Settings.getInstance().isOnlineAccountsMemPoWEnabled()) { + return false; + } + return true; + } + private byte[] getMemoryPoWBytes(byte[] publicKey, long onlineAccountsTimestamp) throws IOException { + byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + outputStream.write(publicKey); + outputStream.write(timestampBytes); + + return outputStream.toByteArray(); + } + + private Integer computeMemoryPoW(byte[] bytes, byte[] publicKey, long onlineAccountsTimestamp) throws TimeoutException { + if (!isMemoryPoWActive()) { + LOGGER.info("Mempow start timestamp not yet reached, and onlineAccountsMemPoWEnabled not enabled in settings"); + return null; + } + + LOGGER.info(String.format("Computing nonce for account %.8s and timestamp %d...", Base58.encode(publicKey), onlineAccountsTimestamp)); + + // Calculate the time until the next online timestamp and use it as a timeout when computing the nonce + Long startTime = NTP.getTime(); + final long nextOnlineAccountsTimestamp = toOnlineAccountTimestamp(startTime) + getOnlineTimestampModulus(); + long timeUntilNextTimestamp = nextOnlineAccountsTimestamp - startTime; + + Integer nonce = MemoryPoW.compute2(bytes, POW_BUFFER_SIZE, POW_DIFFICULTY, timeUntilNextTimestamp); + + double totalSeconds = (NTP.getTime() - startTime) / 1000.0f; + int minutes = (int) ((totalSeconds % 3600) / 60); + int seconds = (int) (totalSeconds % 60); + double hashRate = nonce / totalSeconds; + + LOGGER.info(String.format("Computed nonce for timestamp %d and account %.8s: %d. Buffer size: %d. Difficulty: %d. " + + "Time taken: %02d:%02d. Hashrate: %f", onlineAccountsTimestamp, Base58.encode(publicKey), + nonce, POW_BUFFER_SIZE, POW_DIFFICULTY, minutes, seconds, hashRate)); + + return nonce; + } + + public boolean verifyMemoryPoW(OnlineAccountData onlineAccountData) { + if (!isMemoryPoWActive()) { + // Not active yet, so treat it as valid + return true; + } + + int nonce = onlineAccountData.getNonce(); + + byte[] mempowBytes; + try { + mempowBytes = this.getMemoryPoWBytes(onlineAccountData.getPublicKey(), onlineAccountData.getTimestamp()); + } catch (IOException e) { + return false; + } + + // Verify the nonce + return MemoryPoW.verify2(mempowBytes, POW_BUFFER_SIZE, POW_DIFFICULTY, nonce); + } + + /** * Returns whether online accounts manager has any online accounts with timestamp recent enough to be considered currently online. */ From 4ca174fa0bd5e57c3a03edb88c1df7345a094bf8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 2 Jul 2022 13:45:39 +0100 Subject: [PATCH 06/83] Fixed bug in isMemoryPoWActive() which affected the ability to enable mempow via settings. --- .../java/org/qortal/controller/OnlineAccountsManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index a919ed2a..9e9301ca 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -584,10 +584,10 @@ public class OnlineAccountsManager { private boolean isMemoryPoWActive() { Long now = NTP.getTime(); - if (now < BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp() || Settings.getInstance().isOnlineAccountsMemPoWEnabled()) { - return false; + if (now >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp() || Settings.getInstance().isOnlineAccountsMemPoWEnabled()) { + return true; } - return true; + return false; } private byte[] getMemoryPoWBytes(byte[] publicKey, long onlineAccountsTimestamp) throws IOException { byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); From a3febdf00e1e6bb922b264bae03cfb9215d7820c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 2 Jul 2022 17:26:53 +0100 Subject: [PATCH 07/83] Pass timestamp to OnlineAccountsManager.isMemoryPoWActive() so that block timestamp can be used. --- .../controller/OnlineAccountsManager.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 9e9301ca..332fac18 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -302,7 +302,7 @@ public class OnlineAccountsManager { // Validate mempow if feature trigger is active if (now >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { - if (!getInstance().verifyMemoryPoW(onlineAccountData)) { + if (!getInstance().verifyMemoryPoW(onlineAccountData, now)) { LOGGER.trace(() -> String.format("Rejecting online reward-share for account %s due to invalid PoW nonce", mintingAccount.getAddress())); return false; } @@ -524,7 +524,7 @@ public class OnlineAccountsManager { // Compute nonce Integer nonce; - if (isMemoryPoWActive()) { + if (isMemoryPoWActive(NTP.getTime())) { try { nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp); if (nonce == null) { @@ -549,7 +549,7 @@ public class OnlineAccountsManager { OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce); // Make sure to verify before adding - if (verifyMemoryPoW(ourOnlineAccountData)) { + if (verifyMemoryPoW(ourOnlineAccountData, NTP.getTime())) { ourOnlineAccounts.add(ourOnlineAccountData); } } @@ -582,9 +582,8 @@ public class OnlineAccountsManager { // MemoryPoW - private boolean isMemoryPoWActive() { - Long now = NTP.getTime(); - if (now >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp() || Settings.getInstance().isOnlineAccountsMemPoWEnabled()) { + private boolean isMemoryPoWActive(Long timestamp) { + if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp() || Settings.getInstance().isOnlineAccountsMemPoWEnabled()) { return true; } return false; @@ -600,7 +599,7 @@ public class OnlineAccountsManager { } private Integer computeMemoryPoW(byte[] bytes, byte[] publicKey, long onlineAccountsTimestamp) throws TimeoutException { - if (!isMemoryPoWActive()) { + if (!isMemoryPoWActive(NTP.getTime())) { LOGGER.info("Mempow start timestamp not yet reached, and onlineAccountsMemPoWEnabled not enabled in settings"); return null; } @@ -626,8 +625,8 @@ public class OnlineAccountsManager { return nonce; } - public boolean verifyMemoryPoW(OnlineAccountData onlineAccountData) { - if (!isMemoryPoWActive()) { + public boolean verifyMemoryPoW(OnlineAccountData onlineAccountData, Long timestamp) { + if (!isMemoryPoWActive(timestamp)) { // Not active yet, so treat it as valid return true; } From fe2c63e8e40904546e462510e404265baced6e71 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 2 Jul 2022 17:30:31 +0100 Subject: [PATCH 08/83] Generate random nonces for test accounts. These don't have to be valid for unit tests, because they are treated as "cached already valid accounts" in the block validation. --- .../java/org/qortal/controller/OnlineAccountsManager.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 332fac18..d6ff19d5 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -152,6 +152,7 @@ public class OnlineAccountsManager { byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); final boolean useAggregateCompatibleSignature = onlineAccountsTimestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp(); + final boolean mempowActive = onlineAccountsTimestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp(); Set replacementAccounts = new HashSet<>(); for (PrivateKeyAccount onlineAccount : onlineAccounts) { @@ -162,7 +163,9 @@ public class OnlineAccountsManager { : onlineAccount.sign(timestampBytes); byte[] publicKey = onlineAccount.getPublicKey(); - OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey); + Integer nonce = mempowActive ? new Random().nextInt(500000) : null; + + OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce); replacementAccounts.add(ourOnlineAccountData); } From 020e59743b16b68b9e0095712eb58a6c4d03d2ac Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 10 Jul 2022 19:49:24 +0100 Subject: [PATCH 09/83] Fixed failing test(s) due to merge. --- src/test/resources/test-chain-v2-no-sig-agg.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/resources/test-chain-v2-no-sig-agg.json b/src/test/resources/test-chain-v2-no-sig-agg.json index 75c5528c..71e1cc3d 100644 --- a/src/test/resources/test-chain-v2-no-sig-agg.json +++ b/src/test/resources/test-chain-v2-no-sig-agg.json @@ -52,6 +52,7 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, + "rewardShareLimitTimestamp": 9999999999999, "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, "transactionV6Timestamp": 0, From fbcc870d36b2554850a91bc3847c9aa786ced71f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 15 Jul 2022 10:23:14 +0100 Subject: [PATCH 10/83] Added informational test to compare ConsiceSet size against an int array for online account nonce arrays. --- .../java/org/qortal/test/SchnorrTests.java | 28 ++------------ .../org/qortal/test/common/AccountUtils.java | 38 ++++++++++++++++++- .../test/network/OnlineAccountsTests.java | 34 ++++++++++++++++- 3 files changed, 71 insertions(+), 29 deletions(-) diff --git a/src/test/java/org/qortal/test/SchnorrTests.java b/src/test/java/org/qortal/test/SchnorrTests.java index 03c92d2f..e0d1f1c9 100644 --- a/src/test/java/org/qortal/test/SchnorrTests.java +++ b/src/test/java/org/qortal/test/SchnorrTests.java @@ -8,6 +8,7 @@ import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; import org.junit.Test; import org.qortal.crypto.Qortal25519Extras; import org.qortal.data.network.OnlineAccountData; +import org.qortal.test.common.AccountUtils; import org.qortal.transform.Transformer; import java.math.BigInteger; @@ -28,8 +29,6 @@ public class SchnorrTests extends Qortal25519Extras { Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); } - private static final SecureRandom SECURE_RANDOM = new SecureRandom(); - @Test public void testConversion() { // Scalar form @@ -130,7 +129,7 @@ public class SchnorrTests extends Qortal25519Extras { @Test public void testSimpleAggregate() { - List onlineAccounts = generateOnlineAccounts(1); + List onlineAccounts = AccountUtils.generateOnlineAccounts(1); byte[] aggregatePublicKey = aggregatePublicKeys(onlineAccounts.stream().map(OnlineAccountData::getPublicKey).collect(Collectors.toUnmodifiableList())); System.out.printf("Aggregate public key: %s%n", HashCode.fromBytes(aggregatePublicKey)); @@ -151,7 +150,7 @@ public class SchnorrTests extends Qortal25519Extras { @Test public void testMultipleAggregate() { - List onlineAccounts = generateOnlineAccounts(5000); + List onlineAccounts = AccountUtils.generateOnlineAccounts(5000); byte[] aggregatePublicKey = aggregatePublicKeys(onlineAccounts.stream().map(OnlineAccountData::getPublicKey).collect(Collectors.toUnmodifiableList())); System.out.printf("Aggregate public key: %s%n", HashCode.fromBytes(aggregatePublicKey)); @@ -166,25 +165,4 @@ public class SchnorrTests extends Qortal25519Extras { byte[] timestampBytes = Longs.toByteArray(timestamp); assertTrue(verifyAggregated(aggregatePublicKey, aggregateSignature, timestampBytes)); } - - private List generateOnlineAccounts(int numAccounts) { - List onlineAccounts = new ArrayList<>(); - - long timestamp = System.currentTimeMillis(); - byte[] timestampBytes = Longs.toByteArray(timestamp); - - for (int a = 0; a < numAccounts; ++a) { - byte[] privateKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; - SECURE_RANDOM.nextBytes(privateKey); - - byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; - Qortal25519Extras.generatePublicKey(privateKey, 0, publicKey, 0); - - byte[] signature = signForAggregation(privateKey, timestampBytes); - - onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey)); - } - - return onlineAccounts; - } } diff --git a/src/test/java/org/qortal/test/common/AccountUtils.java b/src/test/java/org/qortal/test/common/AccountUtils.java index ffc4a7a1..0d0b6d6a 100644 --- a/src/test/java/org/qortal/test/common/AccountUtils.java +++ b/src/test/java/org/qortal/test/common/AccountUtils.java @@ -1,12 +1,17 @@ package org.qortal.test.common; import static org.junit.Assert.assertEquals; +import static org.qortal.crypto.Qortal25519Extras.signForAggregation; -import java.util.HashMap; -import java.util.Map; +import java.security.SecureRandom; +import java.util.*; +import com.google.common.primitives.Longs; import org.qortal.account.PrivateKeyAccount; +import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; +import org.qortal.crypto.Qortal25519Extras; +import org.qortal.data.network.OnlineAccountData; import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.PaymentTransactionData; import org.qortal.data.transaction.RewardShareTransactionData; @@ -14,6 +19,7 @@ import org.qortal.data.transaction.TransactionData; import org.qortal.group.Group; import org.qortal.repository.DataException; import org.qortal.repository.Repository; +import org.qortal.transform.Transformer; import org.qortal.utils.Amounts; public class AccountUtils { @@ -21,6 +27,8 @@ public class AccountUtils { public static final int txGroupId = Group.NO_GROUP; public static final long fee = 1L * Amounts.MULTIPLIER; + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); + public static void pay(Repository repository, String testSenderName, String testRecipientName, long amount) throws DataException { PrivateKeyAccount sendingAccount = Common.getTestAccount(repository, testSenderName); PrivateKeyAccount recipientAccount = Common.getTestAccount(repository, testRecipientName); @@ -109,4 +117,30 @@ public class AccountUtils { assertEquals(String.format("%s's %s [%d] balance incorrect", accountName, assetName, assetId), expectedBalance, actualBalance); } + + public static List generateOnlineAccounts(int numAccounts) { + List onlineAccounts = new ArrayList<>(); + + long timestamp = System.currentTimeMillis(); + byte[] timestampBytes = Longs.toByteArray(timestamp); + + final boolean mempowActive = timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp(); + + for (int a = 0; a < numAccounts; ++a) { + byte[] privateKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + SECURE_RANDOM.nextBytes(privateKey); + + byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + Qortal25519Extras.generatePublicKey(privateKey, 0, publicKey, 0); + + byte[] signature = signForAggregation(privateKey, timestampBytes); + + Integer nonce = mempowActive ? new Random().nextInt(500000) : null; + + onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey, nonce)); + } + + return onlineAccounts; + } + } diff --git a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java index 0b554b6a..c5a3115b 100644 --- a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java +++ b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java @@ -1,26 +1,29 @@ package org.qortal.test.network; +import com.google.common.primitives.Ints; +import io.druid.extendedset.intset.ConciseSet; import org.apache.commons.lang3.reflect.FieldUtils; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; -import org.qortal.account.PrivateKeyAccount; import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.controller.BlockMinter; -import org.qortal.controller.OnlineAccountsManager; import org.qortal.data.network.OnlineAccountData; import org.qortal.network.message.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; +import org.qortal.test.common.AccountUtils; import org.qortal.test.common.Common; import org.qortal.transform.Transformer; import org.qortal.utils.Base58; import org.qortal.utils.NTP; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.security.Security; @@ -202,4 +205,31 @@ public class OnlineAccountsTests extends Common { assertTrue(onlineAccountSignatures.size() >= 1 && onlineAccountSignatures.size() <= 3); } } + + @Test + @Ignore(value = "For informational use") + public void testOnlineAccountNonceCompression() throws IOException { + List onlineAccounts = AccountUtils.generateOnlineAccounts(5000); + + // Build array of nonce values + List accountNonces = new ArrayList<>(); + for (OnlineAccountData onlineAccountData : onlineAccounts) { + accountNonces.add(onlineAccountData.getNonce()); + } + + // Write nonces into ConciseSet + ConciseSet nonceSet = new ConciseSet(); + nonceSet = nonceSet.convert(accountNonces); + byte[] conciseEncodedNonces = nonceSet.toByteBuffer().array(); + + // Also write to regular byte array of ints, for comparison + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + for (Integer nonce : accountNonces) { + bytes.write(Ints.toByteArray(nonce)); + } + byte[] standardEncodedNonces = bytes.toByteArray(); + + System.out.println(String.format("Standard: %d", standardEncodedNonces.length)); + System.out.println(String.format("Concise: %d", conciseEncodedNonces.length)); + } } From 46c40ca9cab3a2279871586eedecc7681c15f899 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 20 Jul 2022 10:27:09 +0100 Subject: [PATCH 11/83] Committed stashed code that is functional but probably too messy for production use. Online account nonces are appended to the onlineAccountsSignatures to avoid the need for a new field, but it probably makes more sense to separate them. --- src/main/java/org/qortal/block/Block.java | 71 +++++++++++++++++-- .../transform/block/BlockTransformer.java | 40 +++++++++++ src/main/resources/blockchain.json | 2 +- .../org/qortal/test/BlockArchiveTests.java | 4 +- 4 files changed, 107 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index ddfe247a..fca6fa84 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -10,7 +10,6 @@ import java.math.BigInteger; import java.math.RoundingMode; import java.nio.charset.StandardCharsets; import java.text.DecimalFormat; -import java.text.MessageFormat; import java.text.NumberFormat; import java.util.*; import java.util.stream.Collectors; @@ -90,7 +89,8 @@ public class Block { ONLINE_ACCOUNT_UNKNOWN(71), ONLINE_ACCOUNT_SIGNATURES_MISSING(72), ONLINE_ACCOUNT_SIGNATURES_MALFORMED(73), - ONLINE_ACCOUNT_SIGNATURE_INCORRECT(74); + ONLINE_ACCOUNT_SIGNATURE_INCORRECT(74), + ONLINE_ACCOUNT_NONCE_INCORRECT(75); public final int value; @@ -409,6 +409,31 @@ public class Block { } } + // Add nonces to the end of the online accounts signatures if mempow is active + if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + try { + // Create ordered list of nonce values + List nonces = new ArrayList<>(); + for (int i = 0; i < onlineAccountsCount; ++i) { + Integer accountIndex = accountIndexes.get(i); + OnlineAccountData onlineAccountData = indexedOnlineAccounts.get(accountIndex); + nonces.add(onlineAccountData.getNonce()); + } + + // Encode the nonces to a byte array + byte[] encodedNonces = BlockTransformer.encodeOnlineAccountNonces(nonces); + + // Append the encoded nonces to the encoded online account signatures + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + outputStream.write(onlineAccountsSignatures); + outputStream.write(encodedNonces); + onlineAccountsSignatures = outputStream.toByteArray(); + } + catch (TransformationException | IOException e) { + return null; + } + } + byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData, minter.getPublicKey(), encodedOnlineAccounts)); @@ -1016,12 +1041,15 @@ public class Block { if (this.blockData.getOnlineAccountsSignatures() == null || this.blockData.getOnlineAccountsSignatures().length == 0) return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MISSING; - if (this.blockData.getTimestamp() >= BlockChain.getInstance().getAggregateSignatureTimestamp()) { - // We expect just the one, aggregated signature - if (this.blockData.getOnlineAccountsSignatures().length != Transformer.SIGNATURE_LENGTH) + final int signaturesLength = (this.blockData.getTimestamp() >= BlockChain.getInstance().getAggregateSignatureTimestamp()) ? Transformer.SIGNATURE_LENGTH : onlineRewardShares.size() * Transformer.SIGNATURE_LENGTH; + final int noncesLength = onlineRewardShares.size() * Transformer.INT_LENGTH; + + if (this.blockData.getTimestamp() >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + // We expect nonces to be appended to the online accounts signatures + if (this.blockData.getOnlineAccountsSignatures().length != signaturesLength + noncesLength) return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED; } else { - if (this.blockData.getOnlineAccountsSignatures().length != onlineRewardShares.size() * Transformer.SIGNATURE_LENGTH) + if (this.blockData.getOnlineAccountsSignatures().length != signaturesLength) return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED; } @@ -1029,8 +1057,37 @@ public class Block { long onlineTimestamp = this.blockData.getOnlineAccountsTimestamp(); byte[] onlineTimestampBytes = Longs.toByteArray(onlineTimestamp); + byte[] encodedOnlineAccountSignatures = this.blockData.getOnlineAccountsSignatures(); + + // Split online account signatures into signature(s) + nonces, then validate the nonces + if (this.blockData.getTimestamp() >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + byte[] extractedSignatures = BlockTransformer.extract(encodedOnlineAccountSignatures, 0, signaturesLength); + byte[] extractedNonces = BlockTransformer.extract(encodedOnlineAccountSignatures, signaturesLength, onlineRewardShares.size() * Transformer.INT_LENGTH); + encodedOnlineAccountSignatures = extractedSignatures; + + List nonces = BlockTransformer.decodeOnlineAccountNonces(extractedNonces); + + // Build block's view of online accounts (without signatures, as we don't need them here) + Set onlineAccounts = new HashSet<>(); + for (int i = 0; i < onlineRewardShares.size(); ++i) { + Integer nonce = nonces.get(i); + byte[] publicKey = onlineRewardShares.get(i).getRewardSharePublicKey(); + + OnlineAccountData onlineAccountData = new OnlineAccountData(onlineTimestamp, null, publicKey, nonce); + onlineAccounts.add(onlineAccountData); + } + + // Remove those already validated & cached by online accounts manager - no need to re-validate them + OnlineAccountsManager.getInstance().removeKnown(onlineAccounts, onlineTimestamp); + + // Validate the rest + for (OnlineAccountData onlineAccount : onlineAccounts) + if (!OnlineAccountsManager.getInstance().verifyMemoryPoW(onlineAccount, this.blockData.getTimestamp())) + return ValidationResult.ONLINE_ACCOUNT_NONCE_INCORRECT; + } + // Extract online accounts' timestamp signatures from block data. Only one signature if aggregated. - List onlineAccountsSignatures = BlockTransformer.decodeTimestampSignatures(this.blockData.getOnlineAccountsSignatures()); + List onlineAccountsSignatures = BlockTransformer.decodeTimestampSignatures(encodedOnlineAccountSignatures); if (this.blockData.getTimestamp() >= BlockChain.getInstance().getAggregateSignatureTimestamp()) { // Aggregate all public keys diff --git a/src/main/java/org/qortal/transform/block/BlockTransformer.java b/src/main/java/org/qortal/transform/block/BlockTransformer.java index b61d6900..48e79699 100644 --- a/src/main/java/org/qortal/transform/block/BlockTransformer.java +++ b/src/main/java/org/qortal/transform/block/BlockTransformer.java @@ -478,4 +478,44 @@ public class BlockTransformer extends Transformer { return signatures; } + public static byte[] encodeOnlineAccountNonces(List nonces) throws TransformationException { + try { + final int length = nonces.size() * Transformer.INT_LENGTH; + ByteArrayOutputStream bytes = new ByteArrayOutputStream(length); + + for (int i = 0; i < nonces.size(); ++i) { + Integer nonce = nonces.get(i); + if (nonce == null || nonce < 0) { + throw new TransformationException("Unable to serialize online account nonces due to invalid value"); + } + bytes.write(Ints.toByteArray(nonce)); + } + + return bytes.toByteArray(); + + } catch (IOException e) { + throw new TransformationException("Unable to serialize online account nonces", e); + } + } + + public static List decodeOnlineAccountNonces(byte[] encodedNonces) { + List nonces = new ArrayList<>(); + + ByteBuffer bytes = ByteBuffer.wrap(encodedNonces); + final int count = encodedNonces.length / Transformer.INT_LENGTH; + + for (int i = 0; i < count; i++) { + Integer nonce = bytes.getInt(); + nonces.add(nonce); + } + + return nonces; + } + + public static byte[] extract(byte[] input, int pos, int length) { + byte[] output = new byte[length]; + System.arraycopy(input, pos, output, 0, length); + return output; + } + } diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 03169723..7e906e76 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -72,7 +72,7 @@ }, "genesisInfo": { "version": 4, - "timestamp": "1593450000000", + "timestamp": "1656777099000", "transactions": [ { "type": "ISSUE_ASSET", "assetName": "QORT", "description": "QORTAL coin", "quantity": 0, "isDivisible": true, "data": "{}" }, { "type": "ISSUE_ASSET", "assetName": "Legacy-QORA", "description": "Representative legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true }, diff --git a/src/test/java/org/qortal/test/BlockArchiveTests.java b/src/test/java/org/qortal/test/BlockArchiveTests.java index 32fd0283..3bfa4e84 100644 --- a/src/test/java/org/qortal/test/BlockArchiveTests.java +++ b/src/test/java/org/qortal/test/BlockArchiveTests.java @@ -361,7 +361,7 @@ public class BlockArchiveTests extends Common { assertEquals(0, repository.getBlockArchiveRepository().getBlockArchiveHeight()); // Write blocks 2-900 to the archive (using bulk method) - int fileSizeTarget = 425000; // Pre-calculated size of 900 blocks + int fileSizeTarget = 428600; // Pre-calculated size of 900 blocks assertTrue(HSQLDBDatabaseArchiving.buildBlockArchive(repository, fileSizeTarget)); // Ensure the block archive height has increased @@ -455,7 +455,7 @@ public class BlockArchiveTests extends Common { assertEquals(0, repository.getBlockArchiveRepository().getBlockArchiveHeight()); // Write blocks 2-900 to the archive (using bulk method) - int fileSizeTarget = 42000; // Pre-calculated size of approx 90 blocks + int fileSizeTarget = 42360; // Pre-calculated size of approx 90 blocks assertTrue(HSQLDBDatabaseArchiving.buildBlockArchive(repository, fileSizeTarget)); // Ensure 10 archive files have been created From 85a27c14b864c26c1abacd627497a5cd5f161263 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 20 Jul 2022 10:38:58 +0100 Subject: [PATCH 12/83] Revert incorrect genesis timestamp that somehow made it into the stashed code. --- src/main/resources/blockchain.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 7e906e76..03169723 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -72,7 +72,7 @@ }, "genesisInfo": { "version": 4, - "timestamp": "1656777099000", + "timestamp": "1593450000000", "transactions": [ { "type": "ISSUE_ASSET", "assetName": "QORT", "description": "QORTAL coin", "quantity": 0, "isDivisible": true, "data": "{}" }, { "type": "ISSUE_ASSET", "assetName": "Legacy-QORA", "description": "Representative legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true }, From b9bf945fd8a843cb5154582327f032597a4e40c6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 22 Jul 2022 13:49:19 +0100 Subject: [PATCH 13/83] Removed aggregateSignatureTimestamp. All online account signatures are aggregated - there is no need for backwards support as signatures are trimmed from blocks after 24 hours. testOnlineAccountsModulusV2() had to be removed as this relied on pre-aggregation signatures. --- src/main/java/org/qortal/block/Block.java | 68 ++++----------- .../java/org/qortal/block/BlockChain.java | 6 +- .../controller/OnlineAccountsManager.java | 15 +--- src/main/resources/blockchain.json | 3 +- .../test/network/OnlineAccountsTests.java | 38 +------- .../test-chain-v2-block-timestamps.json | 3 +- .../test-chain-v2-disable-reference.json | 3 +- .../test-chain-v2-founder-rewards.json | 3 +- .../test-chain-v2-leftover-reward.json | 3 +- src/test/resources/test-chain-v2-minting.json | 3 +- .../resources/test-chain-v2-no-sig-agg.json | 87 ------------------- .../test-chain-v2-qora-holder-extremes.json | 3 +- .../resources/test-chain-v2-qora-holder.json | 3 +- .../test-chain-v2-reward-levels.json | 3 +- .../test-chain-v2-reward-scaling.json | 3 +- .../test-chain-v2-reward-shares.json | 3 +- src/test/resources/test-chain-v2.json | 3 +- .../test-settings-v2-no-sig-agg.json | 19 ---- 18 files changed, 34 insertions(+), 235 deletions(-) delete mode 100644 src/test/resources/test-chain-v2-no-sig-agg.json delete mode 100644 src/test/resources/test-settings-v2-no-sig-agg.json diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index fca6fa84..d0f742fd 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -389,25 +389,14 @@ public class Block { byte[] encodedOnlineAccounts = BlockTransformer.encodeOnlineAccounts(onlineAccountsSet); int onlineAccountsCount = onlineAccountsSet.size(); - byte[] onlineAccountsSignatures; - if (timestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp()) { - // Collate all signatures - Collection signaturesToAggregate = indexedOnlineAccounts.values() - .stream() - .map(OnlineAccountData::getSignature) - .collect(Collectors.toList()); + // Collate all signatures + Collection signaturesToAggregate = indexedOnlineAccounts.values() + .stream() + .map(OnlineAccountData::getSignature) + .collect(Collectors.toList()); - // Aggregated, single signature - onlineAccountsSignatures = Qortal25519Extras.aggregateSignatures(signaturesToAggregate); - } else { - // Concatenate online account timestamp signatures (in correct order) - onlineAccountsSignatures = new byte[onlineAccountsCount * Transformer.SIGNATURE_LENGTH]; - for (int i = 0; i < onlineAccountsCount; ++i) { - Integer accountIndex = accountIndexes.get(i); - OnlineAccountData onlineAccountData = indexedOnlineAccounts.get(accountIndex); - System.arraycopy(onlineAccountData.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH); - } - } + // Aggregated, single signature + byte[] onlineAccountsSignatures = Qortal25519Extras.aggregateSignatures(signaturesToAggregate); // Add nonces to the end of the online accounts signatures if mempow is active if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { @@ -1041,7 +1030,7 @@ public class Block { if (this.blockData.getOnlineAccountsSignatures() == null || this.blockData.getOnlineAccountsSignatures().length == 0) return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MISSING; - final int signaturesLength = (this.blockData.getTimestamp() >= BlockChain.getInstance().getAggregateSignatureTimestamp()) ? Transformer.SIGNATURE_LENGTH : onlineRewardShares.size() * Transformer.SIGNATURE_LENGTH; + final int signaturesLength = Transformer.SIGNATURE_LENGTH; final int noncesLength = onlineRewardShares.size() * Transformer.INT_LENGTH; if (this.blockData.getTimestamp() >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { @@ -1089,41 +1078,18 @@ public class Block { // Extract online accounts' timestamp signatures from block data. Only one signature if aggregated. List onlineAccountsSignatures = BlockTransformer.decodeTimestampSignatures(encodedOnlineAccountSignatures); - if (this.blockData.getTimestamp() >= BlockChain.getInstance().getAggregateSignatureTimestamp()) { - // Aggregate all public keys - Collection publicKeys = onlineRewardShares.stream() - .map(RewardShareData::getRewardSharePublicKey) - .collect(Collectors.toList()); + // Aggregate all public keys + Collection publicKeys = onlineRewardShares.stream() + .map(RewardShareData::getRewardSharePublicKey) + .collect(Collectors.toList()); - byte[] aggregatePublicKey = Qortal25519Extras.aggregatePublicKeys(publicKeys); + byte[] aggregatePublicKey = Qortal25519Extras.aggregatePublicKeys(publicKeys); - byte[] aggregateSignature = onlineAccountsSignatures.get(0); + byte[] aggregateSignature = onlineAccountsSignatures.get(0); - // One-step verification of aggregate signature using aggregate public key - if (!Qortal25519Extras.verifyAggregated(aggregatePublicKey, aggregateSignature, onlineTimestampBytes)) - return ValidationResult.ONLINE_ACCOUNT_SIGNATURE_INCORRECT; - } else { - // Build block's view of online accounts - Set onlineAccounts = new HashSet<>(); - for (int i = 0; i < onlineAccountsSignatures.size(); ++i) { - byte[] signature = onlineAccountsSignatures.get(i); - byte[] publicKey = onlineRewardShares.get(i).getRewardSharePublicKey(); - - OnlineAccountData onlineAccountData = new OnlineAccountData(onlineTimestamp, signature, publicKey); - onlineAccounts.add(onlineAccountData); - } - - // Remove those already validated & cached by online accounts manager - no need to re-validate them - OnlineAccountsManager.getInstance().removeKnown(onlineAccounts, onlineTimestamp); - - // Validate the rest - for (OnlineAccountData onlineAccount : onlineAccounts) - if (!Crypto.verify(onlineAccount.getPublicKey(), onlineAccount.getSignature(), onlineTimestampBytes)) - return ValidationResult.ONLINE_ACCOUNT_SIGNATURE_INCORRECT; - - // We've validated these, so allow online accounts manager to cache - OnlineAccountsManager.getInstance().addBlocksOnlineAccounts(onlineAccounts, onlineTimestamp); - } + // One-step verification of aggregate signature using aggregate public key + if (!Qortal25519Extras.verifyAggregated(aggregatePublicKey, aggregateSignature, onlineTimestampBytes)) + return ValidationResult.ONLINE_ACCOUNT_SIGNATURE_INCORRECT; // All online accounts valid, so save our list of online accounts for potential later use this.cachedOnlineRewardShares = onlineRewardShares; diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 56294d1a..0e746766 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -72,8 +72,7 @@ public class BlockChain { calcChainWeightTimestamp, transactionV5Timestamp, transactionV6Timestamp, - disableReferenceTimestamp, - aggregateSignatureTimestamp; + disableReferenceTimestamp; } // Custom transaction fees @@ -445,9 +444,6 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.disableReferenceTimestamp.name()).longValue(); } - public long getAggregateSignatureTimestamp() { - return this.featureTriggers.get(FeatureTrigger.aggregateSignatureTimestamp.name()).longValue(); - } // More complex getters for aspects that change by height or timestamp diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index abd616e7..9cfae61d 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -151,16 +151,13 @@ public class OnlineAccountsManager { return; byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); - final boolean useAggregateCompatibleSignature = onlineAccountsTimestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp(); final boolean mempowActive = onlineAccountsTimestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp(); Set replacementAccounts = new HashSet<>(); for (PrivateKeyAccount onlineAccount : onlineAccounts) { // Check mintingAccount is actually reward-share? - byte[] signature = useAggregateCompatibleSignature - ? Qortal25519Extras.signForAggregation(onlineAccount.getPrivateKey(), timestampBytes) - : onlineAccount.sign(timestampBytes); + byte[] signature = Qortal25519Extras.signForAggregation(onlineAccount.getPrivateKey(), timestampBytes); byte[] publicKey = onlineAccount.getPublicKey(); Integer nonce = mempowActive ? new Random().nextInt(500000) : null; @@ -280,9 +277,7 @@ public class OnlineAccountsManager { // Verify signature byte[] data = Longs.toByteArray(onlineAccountData.getTimestamp()); - boolean isSignatureValid = onlineAccountTimestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp() - ? Qortal25519Extras.verifyAggregated(rewardSharePublicKey, onlineAccountData.getSignature(), data) - : Crypto.verify(rewardSharePublicKey, onlineAccountData.getSignature(), data); + boolean isSignatureValid = Qortal25519Extras.verifyAggregated(rewardSharePublicKey, onlineAccountData.getSignature(), data); if (!isSignatureValid) { LOGGER.trace(() -> String.format("Rejecting invalid online account %s", Base58.encode(rewardSharePublicKey))); return false; @@ -506,8 +501,6 @@ public class OnlineAccountsManager { return false; } - final boolean useAggregateCompatibleSignature = onlineAccountsTimestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp(); - byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); List ourOnlineAccounts = new ArrayList<>(); @@ -544,9 +537,7 @@ public class OnlineAccountsManager { nonce = -1; } - byte[] signature = useAggregateCompatibleSignature - ? Qortal25519Extras.signForAggregation(privateKey, timestampBytes) - : Crypto.sign(privateKey, timestampBytes); + byte[] signature = Qortal25519Extras.signForAggregation(privateKey, timestampBytes); // Our account is online OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce); diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 03169723..38c5d66c 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -67,8 +67,7 @@ "calcChainWeightTimestamp": 1620579600000, "transactionV5Timestamp": 1642176000000, "transactionV6Timestamp": 9999999999999, - "disableReferenceTimestamp": 1655222400000, - "aggregateSignatureTimestamp": 1656864000000 + "disableReferenceTimestamp": 1655222400000 }, "genesisInfo": { "version": 4, diff --git a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java index c5a3115b..c9e646f1 100644 --- a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java +++ b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java @@ -46,7 +46,7 @@ public class OnlineAccountsTests extends Common { @Before public void beforeTest() throws DataException, IOException { - Common.useSettingsAndDb("test-settings-v2-no-sig-agg.json", false); + Common.useSettingsAndDb("test-settings-v2.json", false); NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset()); } @@ -170,42 +170,6 @@ public class OnlineAccountsTests extends Common { } } - @Test - public void testOnlineAccountsModulusV2() throws IllegalAccessException, DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - - // Set feature trigger timestamp to 0 so that it is active - FieldUtils.writeField(BlockChain.getInstance(), "onlineAccountsModulusV2Timestamp", 0L, true); - - List onlineAccountSignatures = new ArrayList<>(); - long fakeNTPOffset = 0L; - - // Mint a block and store its timestamp - Block block = BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share")); - long lastBlockTimestamp = block.getBlockData().getTimestamp(); - - // Mint some blocks and keep track of the different online account signatures - for (int i = 0; i < 30; i++) { - block = BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share")); - - // Increase NTP fixed offset by the block time, to simulate time passing - long blockTimeDelta = block.getBlockData().getTimestamp() - lastBlockTimestamp; - lastBlockTimestamp = block.getBlockData().getTimestamp(); - fakeNTPOffset += blockTimeDelta; - NTP.setFixedOffset(fakeNTPOffset); - - String lastOnlineAccountSignatures58 = Base58.encode(block.getBlockData().getOnlineAccountsSignatures()); - if (!onlineAccountSignatures.contains(lastOnlineAccountSignatures58)) { - onlineAccountSignatures.add(lastOnlineAccountSignatures58); - } - } - - // We expect 1-3 unique signatures over 30 blocks - System.out.println(String.format("onlineAccountSignatures count: %d", onlineAccountSignatures.size())); - assertTrue(onlineAccountSignatures.size() >= 1 && onlineAccountSignatures.size() <= 3); - } - } - @Test @Ignore(value = "For informational use") public void testOnlineAccountNonceCompression() throws IOException { diff --git a/src/test/resources/test-chain-v2-block-timestamps.json b/src/test/resources/test-chain-v2-block-timestamps.json index 38a18a8c..21de096a 100644 --- a/src/test/resources/test-chain-v2-block-timestamps.json +++ b/src/test/resources/test-chain-v2-block-timestamps.json @@ -56,8 +56,7 @@ "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, "transactionV6Timestamp": 9999999999999, - "disableReferenceTimestamp": 9999999999999, - "aggregateSignatureTimestamp": 0 + "disableReferenceTimestamp": 9999999999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-disable-reference.json b/src/test/resources/test-chain-v2-disable-reference.json index 648e91b5..f47a96b3 100644 --- a/src/test/resources/test-chain-v2-disable-reference.json +++ b/src/test/resources/test-chain-v2-disable-reference.json @@ -59,8 +59,7 @@ "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, "transactionV6Timestamp": 0, - "disableReferenceTimestamp": 0, - "aggregateSignatureTimestamp": 0 + "disableReferenceTimestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-founder-rewards.json b/src/test/resources/test-chain-v2-founder-rewards.json index 5a36039c..61b6443c 100644 --- a/src/test/resources/test-chain-v2-founder-rewards.json +++ b/src/test/resources/test-chain-v2-founder-rewards.json @@ -60,8 +60,7 @@ "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, "transactionV6Timestamp": 0, - "disableReferenceTimestamp": 9999999999999, - "aggregateSignatureTimestamp": 0 + "disableReferenceTimestamp": 9999999999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-leftover-reward.json b/src/test/resources/test-chain-v2-leftover-reward.json index 2d53f421..0f8c95d0 100644 --- a/src/test/resources/test-chain-v2-leftover-reward.json +++ b/src/test/resources/test-chain-v2-leftover-reward.json @@ -60,8 +60,7 @@ "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, "transactionV6Timestamp": 0, - "disableReferenceTimestamp": 9999999999999, - "aggregateSignatureTimestamp": 0 + "disableReferenceTimestamp": 9999999999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-minting.json b/src/test/resources/test-chain-v2-minting.json index 87d4efac..eca11cb3 100644 --- a/src/test/resources/test-chain-v2-minting.json +++ b/src/test/resources/test-chain-v2-minting.json @@ -60,8 +60,7 @@ "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, "transactionV6Timestamp": 0, - "disableReferenceTimestamp": 9999999999999, - "aggregateSignatureTimestamp": 0 + "disableReferenceTimestamp": 9999999999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-no-sig-agg.json b/src/test/resources/test-chain-v2-no-sig-agg.json deleted file mode 100644 index 71e1cc3d..00000000 --- a/src/test/resources/test-chain-v2-no-sig-agg.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "isTestChain": true, - "blockTimestampMargin": 500, - "transactionExpiryPeriod": 86400000, - "maxBlockSize": 2097152, - "maxBytesPerUnitFee": 1024, - "unitFee": "0.1", - "nameRegistrationUnitFees": [ - { "timestamp": 1645372800000, "fee": "5" } - ], - "requireGroupForApproval": false, - "minAccountLevelToRewardShare": 5, - "maxRewardSharesPerMintingAccount": 20, - "founderEffectiveMintingLevel": 10, - "onlineAccountSignaturesMinLifetime": 3600000, - "onlineAccountSignaturesMaxLifetime": 86400000, - "onlineAccountsModulusV2Timestamp": 9999999999999, - "rewardsByHeight": [ - { "height": 1, "reward": 100 }, - { "height": 11, "reward": 10 }, - { "height": 21, "reward": 1 } - ], - "sharesByLevel": [ - { "levels": [ 1, 2 ], "share": 0.05 }, - { "levels": [ 3, 4 ], "share": 0.10 }, - { "levels": [ 5, 6 ], "share": 0.15 }, - { "levels": [ 7, 8 ], "share": 0.20 }, - { "levels": [ 9, 10 ], "share": 0.25 } - ], - "qoraHoldersShare": 0.20, - "qoraPerQortReward": 250, - "blocksNeededByLevel": [ 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 ], - "blockTimingsByHeight": [ - { "height": 1, "target": 60000, "deviation": 30000, "power": 0.2 } - ], - "ciyamAtSettings": { - "feePerStep": "0.0001", - "maxStepsPerRound": 500, - "stepsPerFunctionCall": 10, - "minutesPerBlock": 1 - }, - "featureTriggers": { - "messageHeight": 0, - "atHeight": 0, - "assetsTimestamp": 0, - "votingTimestamp": 0, - "arbitraryTimestamp": 0, - "powfixTimestamp": 0, - "qortalTimestamp": 0, - "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0, - "atFindNextTransactionFix": 0, - "newBlockSigHeight": 999999, - "shareBinFix": 999999, - "rewardShareLimitTimestamp": 9999999999999, - "calcChainWeightTimestamp": 0, - "transactionV5Timestamp": 0, - "transactionV6Timestamp": 0, - "disableReferenceTimestamp": 9999999999999, - "aggregateSignatureTimestamp": 9999999999999 - }, - "genesisInfo": { - "version": 4, - "timestamp": 0, - "transactions": [ - { "type": "ISSUE_ASSET", "assetName": "QORT", "description": "QORT native coin", "data": "", "quantity": 0, "isDivisible": true, "fee": 0 }, - { "type": "ISSUE_ASSET", "assetName": "Legacy-QORA", "description": "Representative legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true }, - { "type": "ISSUE_ASSET", "assetName": "QORT-from-QORA", "description": "QORT gained from holding legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true }, - - { "type": "GENESIS", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "amount": "1000000000" }, - { "type": "GENESIS", "recipient": "QixPbJUwsaHsVEofJdozU9zgVqkK6aYhrK", "amount": "1000000" }, - { "type": "GENESIS", "recipient": "QaUpHNhT3Ygx6avRiKobuLdusppR5biXjL", "amount": "1000000" }, - { "type": "GENESIS", "recipient": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "amount": "1000000" }, - - { "type": "CREATE_GROUP", "creatorPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "groupName": "dev-group", "description": "developer group", "isOpen": false, "approvalThreshold": "PCT100", "minimumBlockDelay": 0, "maximumBlockDelay": 1440 }, - - { "type": "ISSUE_ASSET", "issuerPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "assetName": "TEST", "description": "test asset", "data": "", "quantity": "1000000", "isDivisible": true, "fee": 0 }, - { "type": "ISSUE_ASSET", "issuerPublicKey": "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry", "assetName": "OTHER", "description": "other test asset", "data": "", "quantity": "1000000", "isDivisible": true, "fee": 0 }, - { "type": "ISSUE_ASSET", "issuerPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "assetName": "GOLD", "description": "gold test asset", "data": "", "quantity": "1000000", "isDivisible": true, "fee": 0 }, - - { "type": "ACCOUNT_FLAGS", "target": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "andMask": -1, "orMask": 1, "xorMask": 0 }, - { "type": "REWARD_SHARE", "minterPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "rewardSharePublicKey": "7PpfnvLSG7y4HPh8hE7KoqAjLCkv7Ui6xw4mKAkbZtox", "sharePercent": "100" }, - - { "type": "ACCOUNT_LEVEL", "target": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "level": 5 } - ] - } -} diff --git a/src/test/resources/test-chain-v2-qora-holder-extremes.json b/src/test/resources/test-chain-v2-qora-holder-extremes.json index 4d9becf3..9839e215 100644 --- a/src/test/resources/test-chain-v2-qora-holder-extremes.json +++ b/src/test/resources/test-chain-v2-qora-holder-extremes.json @@ -60,8 +60,7 @@ "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, "transactionV6Timestamp": 0, - "disableReferenceTimestamp": 9999999999999, - "aggregateSignatureTimestamp": 0 + "disableReferenceTimestamp": 9999999999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-qora-holder.json b/src/test/resources/test-chain-v2-qora-holder.json index 5c9002da..c02243b2 100644 --- a/src/test/resources/test-chain-v2-qora-holder.json +++ b/src/test/resources/test-chain-v2-qora-holder.json @@ -60,8 +60,7 @@ "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, "transactionV6Timestamp": 0, - "disableReferenceTimestamp": 9999999999999, - "aggregateSignatureTimestamp": 0 + "disableReferenceTimestamp": 9999999999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-reward-levels.json b/src/test/resources/test-chain-v2-reward-levels.json index 90c80c63..70470ffa 100644 --- a/src/test/resources/test-chain-v2-reward-levels.json +++ b/src/test/resources/test-chain-v2-reward-levels.json @@ -60,8 +60,7 @@ "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, "transactionV6Timestamp": 0, - "disableReferenceTimestamp": 9999999999999, - "aggregateSignatureTimestamp": 0 + "disableReferenceTimestamp": 9999999999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-reward-scaling.json b/src/test/resources/test-chain-v2-reward-scaling.json index 228c76f6..4140b3d2 100644 --- a/src/test/resources/test-chain-v2-reward-scaling.json +++ b/src/test/resources/test-chain-v2-reward-scaling.json @@ -60,8 +60,7 @@ "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, "transactionV6Timestamp": 0, - "disableReferenceTimestamp": 9999999999999, - "aggregateSignatureTimestamp": 0 + "disableReferenceTimestamp": 9999999999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-reward-shares.json b/src/test/resources/test-chain-v2-reward-shares.json index 9e713095..2b024825 100644 --- a/src/test/resources/test-chain-v2-reward-shares.json +++ b/src/test/resources/test-chain-v2-reward-shares.json @@ -60,8 +60,7 @@ "newConsensusTimestamp": 0, "transactionV5Timestamp": 0, "transactionV6Timestamp": 0, - "disableReferenceTimestamp": 9999999999999, - "aggregateSignatureTimestamp": 0 + "disableReferenceTimestamp": 9999999999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index b2548f41..07e308e6 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -60,8 +60,7 @@ "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, "transactionV6Timestamp": 0, - "disableReferenceTimestamp": 9999999999999, - "aggregateSignatureTimestamp": 0 + "disableReferenceTimestamp": 9999999999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-settings-v2-no-sig-agg.json b/src/test/resources/test-settings-v2-no-sig-agg.json deleted file mode 100644 index 1a55fa65..00000000 --- a/src/test/resources/test-settings-v2-no-sig-agg.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "repositoryPath": "testdb", - "bitcoinNet": "TEST3", - "litecoinNet": "TEST3", - "restrictedApi": false, - "blockchainConfig": "src/test/resources/test-chain-v2-no-sig-agg.json", - "exportPath": "qortal-backup-test", - "bootstrap": false, - "wipeUnconfirmedOnStart": false, - "testNtpOffset": 0, - "minPeers": 0, - "pruneBlockLimit": 100, - "bootstrapFilenamePrefix": "test-", - "dataPath": "data-test", - "tempDataPath": "data-test/_temp", - "listsPath": "lists-test", - "storagePolicy": "FOLLOWED_OR_VIEWED", - "maxStorageCapacity": 104857600 -} From a9267760ebe56a96a64bab94309735b4e2cc3b2e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 8 Jul 2022 11:12:58 +0100 Subject: [PATCH 14/83] qoraHoldersShare reworked to qoraHoldersShareByHeight. This allows the QORA share percentage to be modified at different heights, based on community votes. Added unit test to simulate a reduction. # Conflicts: # src/test/java/org/qortal/test/minting/RewardTests.java --- src/main/java/org/qortal/block/Block.java | 2 +- .../java/org/qortal/block/BlockChain.java | 30 ++++++++---- src/main/resources/blockchain.json | 5 +- .../org/qortal/test/minting/RewardTests.java | 46 ++++++++++++++++++- .../test-chain-v2-block-timestamps.json | 5 +- .../test-chain-v2-disable-reference.json | 5 +- .../test-chain-v2-founder-rewards.json | 5 +- .../test-chain-v2-leftover-reward.json | 5 +- src/test/resources/test-chain-v2-minting.json | 5 +- .../test-chain-v2-qora-holder-extremes.json | 5 +- .../resources/test-chain-v2-qora-holder.json | 5 +- .../test-chain-v2-reward-levels.json | 5 +- .../test-chain-v2-reward-scaling.json | 5 +- .../test-chain-v2-reward-shares.json | 5 +- src/test/resources/test-chain-v2.json | 5 +- 15 files changed, 113 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 2ef97ef5..e1ad84ff 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1975,7 +1975,7 @@ public class Block { // Fetch list of legacy QORA holders who haven't reached their cap of QORT reward. List qoraHolders = this.repository.getAccountRepository().getEligibleLegacyQoraHolders(isProcessingNotOrphaning ? null : this.blockData.getHeight()); final boolean haveQoraHolders = !qoraHolders.isEmpty(); - final long qoraHoldersShare = BlockChain.getInstance().getQoraHoldersShare(); + final long qoraHoldersShare = BlockChain.getInstance().getQoraHoldersShareAtHeight(this.blockData.getHeight()); // Perform account-level-based reward scaling if appropriate if (!haveFounders) { diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index cddb38cc..793c5210 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -126,9 +126,13 @@ public class BlockChain { /** Generated lookup of share-bin by account level */ private AccountLevelShareBin[] shareBinsByLevel; - /** Share of block reward/fees to legacy QORA coin holders */ - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - private Long qoraHoldersShare; + /** Share of block reward/fees to legacy QORA coin holders, by block height */ + public static class ShareByHeight { + public int height; + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long share; + } + private List qoraHoldersShareByHeight; /** How many legacy QORA per 1 QORT of block reward. */ @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) @@ -382,10 +386,6 @@ public class BlockChain { return this.cumulativeBlocksByLevel; } - public long getQoraHoldersShare() { - return this.qoraHoldersShare; - } - public long getQoraPerQortReward() { return this.qoraPerQortReward; } @@ -504,6 +504,15 @@ public class BlockChain { return 0; } + public long getQoraHoldersShareAtHeight(int ourHeight) { + // Scan through for QORA share at our height + for (int i = qoraHoldersShareByHeight.size() - 1; i >= 0; --i) + if (qoraHoldersShareByHeight.get(i).height <= ourHeight) + return qoraHoldersShareByHeight.get(i).share; + + return 0; + } + /** Validate blockchain config read from JSON */ private void validateConfig() { if (this.genesisInfo == null) @@ -515,8 +524,8 @@ public class BlockChain { if (this.sharesByLevel == null) Settings.throwValidationError("No \"sharesByLevel\" entry found in blockchain config"); - if (this.qoraHoldersShare == null) - Settings.throwValidationError("No \"qoraHoldersShare\" entry found in blockchain config"); + if (this.qoraHoldersShareByHeight == null) + Settings.throwValidationError("No \"qoraHoldersShareByHeight\" entry found in blockchain config"); if (this.qoraPerQortReward == null) Settings.throwValidationError("No \"qoraPerQortReward\" entry found in blockchain config"); @@ -554,7 +563,7 @@ public class BlockChain { Settings.throwValidationError(String.format("Missing feature trigger \"%s\" in blockchain config", featureTrigger.name())); // Check block reward share bounds - long totalShare = this.qoraHoldersShare; + long totalShare = this.getQoraHoldersShareAtHeight(1); // Add share percents for account-level-based rewards for (AccountLevelShareBin accountLevelShareBin : this.sharesByLevel) totalShare += accountLevelShareBin.share; @@ -592,6 +601,7 @@ public class BlockChain { this.blocksNeededByLevel = Collections.unmodifiableList(this.blocksNeededByLevel); this.cumulativeBlocksByLevel = Collections.unmodifiableList(this.cumulativeBlocksByLevel); this.blockTimingsByHeight = Collections.unmodifiableList(this.blockTimingsByHeight); + this.qoraHoldersShareByHeight = Collections.unmodifiableList(this.qoraHoldersShareByHeight); } /** diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 381a280a..6d38ffef 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -46,7 +46,10 @@ { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], - "qoraHoldersShare": 0.20, + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 9999999, "share": 0.01 } + ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, "shareBinActivationMinLevel": 7, diff --git a/src/test/java/org/qortal/test/minting/RewardTests.java b/src/test/java/org/qortal/test/minting/RewardTests.java index 4aee2de1..5e2e2463 100644 --- a/src/test/java/org/qortal/test/minting/RewardTests.java +++ b/src/test/java/org/qortal/test/minting/RewardTests.java @@ -13,6 +13,7 @@ import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; import org.qortal.asset.Asset; +import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.RewardByHeight; import org.qortal.controller.BlockMinter; @@ -108,7 +109,7 @@ public class RewardTests extends Common { public void testLegacyQoraReward() throws DataException { Common.useSettings("test-settings-v2-qora-holder-extremes.json"); - long qoraHoldersShare = BlockChain.getInstance().getQoraHoldersShare(); + long qoraHoldersShare = BlockChain.getInstance().getQoraHoldersShareAtHeight(1); BigInteger qoraHoldersShareBI = BigInteger.valueOf(qoraHoldersShare); long qoraPerQort = BlockChain.getInstance().getQoraPerQortReward(); @@ -189,6 +190,47 @@ public class RewardTests extends Common { } } + @Test + public void testLegacyQoraRewardReduction() throws DataException { + Common.useSettings("test-settings-v2-qora-holder-extremes.json"); + + // Make sure that the QORA share reduces between blocks 4 and 5 + assertTrue(BlockChain.getInstance().getQoraHoldersShareAtHeight(5) < BlockChain.getInstance().getQoraHoldersShareAtHeight(4)); + + // Keep track of balance deltas at each height + Map chloeQortBalanceDeltaAtEachHeight = new HashMap<>(); + + try (final Repository repository = RepositoryManager.getRepository()) { + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + long chloeLastQortBalance = initialBalances.get("chloe").get(Asset.QORT); + + for (int i=2; i<=10; i++) { + + Block block = BlockUtils.mintBlock(repository); + + // Add to map of balance deltas at each height + long chloeNewQortBalance = AccountUtils.getBalance(repository, "chloe", Asset.QORT); + chloeQortBalanceDeltaAtEachHeight.put(block.getBlockData().getHeight(), chloeNewQortBalance - chloeLastQortBalance); + chloeLastQortBalance = chloeNewQortBalance; + } + + // Ensure blocks 2-4 paid out the same rewards to Chloe + assertEquals(chloeQortBalanceDeltaAtEachHeight.get(2), chloeQortBalanceDeltaAtEachHeight.get(4)); + + // Ensure block 5 paid a lower reward + assertTrue(chloeQortBalanceDeltaAtEachHeight.get(5) < chloeQortBalanceDeltaAtEachHeight.get(4)); + + // Check that the reward was 20x lower + assertTrue(chloeQortBalanceDeltaAtEachHeight.get(5) == chloeQortBalanceDeltaAtEachHeight.get(4) / 20); + + // Orphan to block 4 and ensure that Chloe's balance hasn't been incorrectly affected by the reward reduction + BlockUtils.orphanToBlock(repository, 4); + long expectedChloeQortBalance = initialBalances.get("chloe").get(Asset.QORT) + chloeQortBalanceDeltaAtEachHeight.get(2) + + chloeQortBalanceDeltaAtEachHeight.get(3) + chloeQortBalanceDeltaAtEachHeight.get(4); + assertEquals(expectedChloeQortBalance, AccountUtils.getBalance(repository, "chloe", Asset.QORT)); + } + } + /** Use Alice-Chloe reward-share to bump Chloe from level 0 to level 1, then check orphaning works as expected. */ @Test public void testLevel1() throws DataException { @@ -294,7 +336,7 @@ public class RewardTests extends Common { * So Dilbert should receive 100% - legacy QORA holder's share. */ - final long qoraHoldersShare = BlockChain.getInstance().getQoraHoldersShare(); + final long qoraHoldersShare = BlockChain.getInstance().getQoraHoldersShareAtHeight(1); final long remainingShare = 1_00000000 - qoraHoldersShare; long dilbertExpectedBalance = initialBalances.get("dilbert").get(Asset.QORT); diff --git a/src/test/resources/test-chain-v2-block-timestamps.json b/src/test/resources/test-chain-v2-block-timestamps.json index d7beba3c..d041463f 100644 --- a/src/test/resources/test-chain-v2-block-timestamps.json +++ b/src/test/resources/test-chain-v2-block-timestamps.json @@ -26,7 +26,10 @@ { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], - "qoraHoldersShare": 0.20, + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 1000000, "share": 0.01 } + ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, "shareBinActivationMinLevel": 7, diff --git a/src/test/resources/test-chain-v2-disable-reference.json b/src/test/resources/test-chain-v2-disable-reference.json index 3ad7da60..8a4a58cc 100644 --- a/src/test/resources/test-chain-v2-disable-reference.json +++ b/src/test/resources/test-chain-v2-disable-reference.json @@ -30,7 +30,10 @@ { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], - "qoraHoldersShare": 0.20, + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 1000000, "share": 0.01 } + ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, "shareBinActivationMinLevel": 7, diff --git a/src/test/resources/test-chain-v2-founder-rewards.json b/src/test/resources/test-chain-v2-founder-rewards.json index c9d63800..e8f1e52b 100644 --- a/src/test/resources/test-chain-v2-founder-rewards.json +++ b/src/test/resources/test-chain-v2-founder-rewards.json @@ -31,7 +31,10 @@ { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], - "qoraHoldersShare": 0.20, + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 1000000, "share": 0.01 } + ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, "shareBinActivationMinLevel": 7, diff --git a/src/test/resources/test-chain-v2-leftover-reward.json b/src/test/resources/test-chain-v2-leftover-reward.json index ad5a1f9a..233f6aa0 100644 --- a/src/test/resources/test-chain-v2-leftover-reward.json +++ b/src/test/resources/test-chain-v2-leftover-reward.json @@ -31,7 +31,10 @@ { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], - "qoraHoldersShare": 0.20, + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 1000000, "share": 0.01 } + ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, "shareBinActivationMinLevel": 7, diff --git a/src/test/resources/test-chain-v2-minting.json b/src/test/resources/test-chain-v2-minting.json index 1d57e119..b1a25e86 100644 --- a/src/test/resources/test-chain-v2-minting.json +++ b/src/test/resources/test-chain-v2-minting.json @@ -31,7 +31,10 @@ { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], - "qoraHoldersShare": 0.20, + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 1000000, "share": 0.01 } + ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, "shareBinActivationMinLevel": 7, diff --git a/src/test/resources/test-chain-v2-qora-holder-extremes.json b/src/test/resources/test-chain-v2-qora-holder-extremes.json index 97d7a320..a3bd1921 100644 --- a/src/test/resources/test-chain-v2-qora-holder-extremes.json +++ b/src/test/resources/test-chain-v2-qora-holder-extremes.json @@ -31,7 +31,10 @@ { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], - "qoraHoldersShare": 0.20, + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 5, "share": 0.01 } + ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, "shareBinActivationMinLevel": 7, diff --git a/src/test/resources/test-chain-v2-qora-holder.json b/src/test/resources/test-chain-v2-qora-holder.json index 6d0c2223..76d10b0d 100644 --- a/src/test/resources/test-chain-v2-qora-holder.json +++ b/src/test/resources/test-chain-v2-qora-holder.json @@ -31,7 +31,10 @@ { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], - "qoraHoldersShare": 0.20, + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 1000000, "share": 0.01 } + ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, "shareBinActivationMinLevel": 7, diff --git a/src/test/resources/test-chain-v2-reward-levels.json b/src/test/resources/test-chain-v2-reward-levels.json index 03a05a5e..5434345d 100644 --- a/src/test/resources/test-chain-v2-reward-levels.json +++ b/src/test/resources/test-chain-v2-reward-levels.json @@ -31,7 +31,10 @@ { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], - "qoraHoldersShare": 0.20, + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 1000000, "share": 0.01 } + ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 1, "shareBinActivationMinLevel": 7, diff --git a/src/test/resources/test-chain-v2-reward-scaling.json b/src/test/resources/test-chain-v2-reward-scaling.json index a108c651..97736e7e 100644 --- a/src/test/resources/test-chain-v2-reward-scaling.json +++ b/src/test/resources/test-chain-v2-reward-scaling.json @@ -31,7 +31,10 @@ { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], - "qoraHoldersShare": 0.20, + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 1000000, "share": 0.01 } + ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, "shareBinActivationMinLevel": 7, diff --git a/src/test/resources/test-chain-v2-reward-shares.json b/src/test/resources/test-chain-v2-reward-shares.json index 8f14e48f..74ce003c 100644 --- a/src/test/resources/test-chain-v2-reward-shares.json +++ b/src/test/resources/test-chain-v2-reward-shares.json @@ -30,7 +30,10 @@ { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], - "qoraHoldersShare": 0.20, + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 1000000, "share": 0.01 } + ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, "shareBinActivationMinLevel": 7, diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index 3b380d29..138dd5d9 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -31,7 +31,10 @@ { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], - "qoraHoldersShare": 0.20, + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 1000000, "share": 0.01 } + ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, "shareBinActivationMinLevel": 7, From 8b6124771285c57115b248b1c4ceaa3c55786710 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 2 Sep 2022 16:43:46 +0100 Subject: [PATCH 15/83] Added shareBinsByLevelV2. This allows for different share bin distribution starting at an undecided future block height. This height will correspond with the QORA reduction. New values decided in recent community vote. --- src/main/java/org/qortal/block/Block.java | 12 +- .../java/org/qortal/block/BlockChain.java | 79 ++++-- src/main/resources/blockchain.json | 10 +- .../org/qortal/test/minting/RewardTests.java | 246 +++++++++++++++++- .../test-chain-v2-block-timestamps.json | 10 +- .../test-chain-v2-disable-reference.json | 10 +- .../test-chain-v2-founder-rewards.json | 10 +- .../test-chain-v2-leftover-reward.json | 10 +- src/test/resources/test-chain-v2-minting.json | 10 +- .../resources/test-chain-v2-no-sig-agg.json | 20 +- .../test-chain-v2-qora-holder-extremes.json | 19 +- .../test-chain-v2-qora-holder-reduction.json | 107 ++++++++ .../resources/test-chain-v2-qora-holder.json | 10 +- .../test-chain-v2-reward-levels.json | 12 +- .../test-chain-v2-reward-scaling.json | 10 +- .../test-chain-v2-reward-shares.json | 10 +- src/test/resources/test-chain-v2.json | 10 +- ...est-settings-v2-qora-holder-reduction.json | 11 + 18 files changed, 559 insertions(+), 47 deletions(-) create mode 100644 src/test/resources/test-chain-v2-qora-holder-reduction.json create mode 100644 src/test/resources/test-settings-v2-qora-holder-reduction.json diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index e1ad84ff..2bd6dda0 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -10,7 +10,6 @@ import java.math.BigInteger; import java.math.RoundingMode; import java.nio.charset.StandardCharsets; import java.text.DecimalFormat; -import java.text.MessageFormat; import java.text.NumberFormat; import java.util.*; import java.util.stream.Collectors; @@ -185,8 +184,11 @@ public class Block { if (accountLevel <= 0) return null; // level 0 isn't included in any share bins + // Select the correct set of share bins based on block height final BlockChain blockChain = BlockChain.getInstance(); - final AccountLevelShareBin[] shareBinsByLevel = blockChain.getShareBinsByAccountLevel(); + final AccountLevelShareBin[] shareBinsByLevel = (blockHeight >= blockChain.getSharesByLevelV2Height()) ? + blockChain.getShareBinsByAccountLevelV2() : blockChain.getShareBinsByAccountLevelV1(); + if (accountLevel > shareBinsByLevel.length) return null; @@ -1896,10 +1898,14 @@ public class Block { final List onlineFounderAccounts = expandedAccounts.stream().filter(expandedAccount -> expandedAccount.isMinterFounder).collect(Collectors.toList()); final boolean haveFounders = !onlineFounderAccounts.isEmpty(); + // Select the correct set of share bins based on block height + List accountLevelShareBinsForBlock = (this.blockData.getHeight() >= BlockChain.getInstance().getSharesByLevelV2Height()) ? + BlockChain.getInstance().getAccountLevelShareBinsV2() : BlockChain.getInstance().getAccountLevelShareBinsV1(); + // Determine reward candidates based on account level // This needs a deep copy, so the shares can be modified when tiers aren't activated yet List accountLevelShareBins = new ArrayList<>(); - for (AccountLevelShareBin accountLevelShareBin : BlockChain.getInstance().getAccountLevelShareBins()) { + for (AccountLevelShareBin accountLevelShareBin : accountLevelShareBinsForBlock) { accountLevelShareBins.add((AccountLevelShareBin) accountLevelShareBin.clone()); } diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 793c5210..a95af2b9 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -68,6 +68,7 @@ public class BlockChain { atFindNextTransactionFix, newBlockSigHeight, shareBinFix, + sharesByLevelV2Height, rewardShareLimitTimestamp, calcChainWeightTimestamp, transactionV5Timestamp, @@ -122,9 +123,11 @@ public class BlockChain { return shareBinCopy; } } - private List sharesByLevel; + private List sharesByLevelV1; + private List sharesByLevelV2; /** Generated lookup of share-bin by account level */ - private AccountLevelShareBin[] shareBinsByLevel; + private AccountLevelShareBin[] shareBinsByLevelV1; + private AccountLevelShareBin[] shareBinsByLevelV2; /** Share of block reward/fees to legacy QORA coin holders, by block height */ public static class ShareByHeight { @@ -370,12 +373,20 @@ public class BlockChain { return this.rewardsByHeight; } - public List getAccountLevelShareBins() { - return this.sharesByLevel; + public List getAccountLevelShareBinsV1() { + return this.sharesByLevelV1; } - public AccountLevelShareBin[] getShareBinsByAccountLevel() { - return this.shareBinsByLevel; + public List getAccountLevelShareBinsV2() { + return this.sharesByLevelV2; + } + + public AccountLevelShareBin[] getShareBinsByAccountLevelV1() { + return this.shareBinsByLevelV1; + } + + public AccountLevelShareBin[] getShareBinsByAccountLevelV2() { + return this.shareBinsByLevelV2; } public List getBlocksNeededByLevel() { @@ -444,6 +455,10 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.shareBinFix.name()).intValue(); } + public int getSharesByLevelV2Height() { + return this.featureTriggers.get(FeatureTrigger.sharesByLevelV2Height.name()).intValue(); + } + public long getRewardShareLimitTimestamp() { return this.featureTriggers.get(FeatureTrigger.rewardShareLimitTimestamp.name()).longValue(); } @@ -521,8 +536,11 @@ public class BlockChain { if (this.rewardsByHeight == null) Settings.throwValidationError("No \"rewardsByHeight\" entry found in blockchain config"); - if (this.sharesByLevel == null) - Settings.throwValidationError("No \"sharesByLevel\" entry found in blockchain config"); + if (this.sharesByLevelV1 == null) + Settings.throwValidationError("No \"sharesByLevelV1\" entry found in blockchain config"); + + if (this.sharesByLevelV2 == null) + Settings.throwValidationError("No \"sharesByLevelV2\" entry found in blockchain config"); if (this.qoraHoldersShareByHeight == null) Settings.throwValidationError("No \"qoraHoldersShareByHeight\" entry found in blockchain config"); @@ -562,13 +580,22 @@ public class BlockChain { if (!this.featureTriggers.containsKey(featureTrigger.name())) Settings.throwValidationError(String.format("Missing feature trigger \"%s\" in blockchain config", featureTrigger.name())); - // Check block reward share bounds - long totalShare = this.getQoraHoldersShareAtHeight(1); + // Check block reward share bounds (V1) + long totalShareV1 = this.qoraHoldersShareByHeight.get(0).share; // Add share percents for account-level-based rewards - for (AccountLevelShareBin accountLevelShareBin : this.sharesByLevel) - totalShare += accountLevelShareBin.share; + for (AccountLevelShareBin accountLevelShareBin : this.sharesByLevelV1) + totalShareV1 += accountLevelShareBin.share; - if (totalShare < 0 || totalShare > 1_00000000L) + if (totalShareV1 < 0 || totalShareV1 > 1_00000000L) + Settings.throwValidationError("Total non-founder share out of bounds (0 1_00000000L) Settings.throwValidationError("Total non-founder share out of bounds (0 mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + byte[] chloeRewardSharePrivateKey; + // Bob self-share NOT online + + // Mint some blocks, to get us close to the V2 activation, but with some room for Chloe and Dilbert to start minting some blocks + for (int i=0; i<990; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Chloe self share comes online + try { + chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + } catch (IllegalArgumentException ex) { + LOGGER.error("FAILED {}", ex.getLocalizedMessage(), ex); + throw ex; + } + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share comes online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint 6 more blocks, so that V2 share bins are nearly activated + for (int i=0; i<6; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure that the levels are as we expect + assertEquals(10, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(2, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Ensure that only Alice is a founder + assertEquals(1, getFlags(repository, "alice")); + assertEquals(0, getFlags(repository, "bob")); + assertEquals(0, getFlags(repository, "chloe")); + assertEquals(0, getFlags(repository, "dilbert")); + + // Now that everyone is at level 1 or 2, we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are at the correct height and block reward value + assertEquals(1000, (int) repository.getBlockRepository().getLastBlock().getHeight()); + assertEquals(100000000L, blockReward); + + // We are past the sharesByLevelV2Height feature trigger, so we expect level 1 and 2 to share the increased reward (6%) + final int level1And2SharePercent = 6_00; // 6% + final long level1And2ShareAmount = (blockReward * level1And2SharePercent) / 100L / 100L; + final long expectedLevel1And2RewardV2 = level1And2ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderRewardV2 = blockReward - level1And2ShareAmount; // Alice should receive the remainder + + // Validate the balances + assertEquals(6000000, level1And2ShareAmount); + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderRewardV2); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedLevel1And2RewardV2); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedLevel1And2RewardV2); + + // Now orphan the latest block. This brings us to the threshold of the sharesByLevelV2Height feature trigger + BlockUtils.orphanBlocks(repository, 1); + assertEquals(999, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Ensure the latest block rewards have been subtracted and they have returned to their initial values + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance); + + // Orphan another block. This time, the block that was orphaned was prior to the sharesByLevelV2Height feature trigger. + BlockUtils.orphanBlocks(repository, 1); + assertEquals(998, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // In V1 of share-bins, level 1-2 pays out 5% instead of 6% + final int level1And2SharePercentV1 = 5_00; // 5% + final long level1And2ShareAmountV1 = (blockReward * level1And2SharePercentV1) / 100L / 100L; + final long expectedLevel1And2RewardV1 = level1And2ShareAmountV1 / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderRewardV1 = blockReward - level1And2ShareAmountV1; // Alice should receive the remainder + + // Validate the share amounts and balances + assertEquals(5000000, level1And2ShareAmountV1); + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance-expectedFounderRewardV1); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance-expectedLevel1And2RewardV1); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance-expectedLevel1And2RewardV1); + + // Orphan the latest block one last time + BlockUtils.orphanBlocks(repository, 1); + assertEquals(997, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Validate balances + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance-(expectedFounderRewardV1*2)); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance-(expectedLevel1And2RewardV1*2)); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance-(expectedLevel1And2RewardV1*2)); + + } + } + + /** Test rewards for level 1 and 2 accounts with V2 share-bin layout (post QORA reduction) + * plus some legacy QORA holders */ + @Test + public void testLevel1And2RewardsShareBinsV2WithQoraHolders() throws DataException { + Common.useSettings("test-settings-v2-qora-holder-extremes.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Some legacy QORA holders exist (Bob and Chloe) + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + byte[] chloeRewardSharePrivateKey; + // Bob self-share NOT online + + // Mint some blocks, to get us close to the V2 activation, but with some room for Chloe and Dilbert to start minting some blocks + for (int i=0; i<990; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Chloe self share comes online + try { + chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + } catch (IllegalArgumentException ex) { + LOGGER.error("FAILED {}", ex.getLocalizedMessage(), ex); + throw ex; + } + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share comes online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint 6 more blocks, so that V2 share bins are nearly activated + for (int i=0; i<6; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure that the levels are as we expect + assertEquals(10, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(2, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Ensure that only Alice is a founder + assertEquals(1, getFlags(repository, "alice")); + assertEquals(0, getFlags(repository, "bob")); + assertEquals(0, getFlags(repository, "chloe")); + assertEquals(0, getFlags(repository, "dilbert")); + + // Now that everyone is at level 1 or 2, we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are at the correct height and block reward value + assertEquals(1000, (int) repository.getBlockRepository().getLastBlock().getHeight()); + assertEquals(100000000L, blockReward); + + // We are past the sharesByLevelV2Height feature trigger, so we expect level 1 and 2 to share the increased reward (6%) + // and the QORA share will be 1% + final int level1And2SharePercent = 6_00; // 6% + final int qoraSharePercentV2 = 1_00; // 1% + final long qoraShareAmountV2 = (blockReward * qoraSharePercentV2) / 100L / 100L; + final long level1And2ShareAmount = (blockReward * level1And2SharePercent) / 100L / 100L; + final long expectedLevel1And2RewardV2 = level1And2ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderRewardV2 = blockReward - level1And2ShareAmount - qoraShareAmountV2; // Alice should receive the remainder + + // Validate the balances + assertEquals(6000000, level1And2ShareAmount); + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderRewardV2); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + // Chloe is a QORA holder and will receive additional QORT, so it's not easy to pre-calculate her balance + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedLevel1And2RewardV2); + + // Now orphan the latest block. This brings us to the threshold of the sharesByLevelV2Height feature trigger + BlockUtils.orphanBlocks(repository, 1); + assertEquals(999, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Ensure the latest block rewards have been subtracted and they have returned to their initial values + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance); + + // Orphan another block. This time, the block that was orphaned was prior to the sharesByLevelV2Height feature trigger. + BlockUtils.orphanBlocks(repository, 1); + assertEquals(998, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // In V1 of share-bins, level 1-2 pays out 5% instead of 6%, and the QORA share is higher at 20% + final int level1And2SharePercentV1 = 5_00; // 5% + final int qoraSharePercentV1 = 20_00; // 20% + final long qoraShareAmountV1 = (blockReward * qoraSharePercentV1) / 100L / 100L; + final long level1And2ShareAmountV1 = (blockReward * level1And2SharePercentV1) / 100L / 100L; + final long expectedLevel1And2RewardV1 = level1And2ShareAmountV1 / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderRewardV1 = blockReward - level1And2ShareAmountV1 - qoraShareAmountV1; // Alice should receive the remainder + + // Validate the share amounts and balances + assertEquals(5000000, level1And2ShareAmountV1); + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance-expectedFounderRewardV1); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + // Chloe is a QORA holder and will receive additional QORT, so it's not easy to pre-calculate her balance + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance-expectedLevel1And2RewardV1); + + // Orphan the latest block one last time + BlockUtils.orphanBlocks(repository, 1); + assertEquals(997, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Validate balances + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance-(expectedFounderRewardV1*2)); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + // Chloe is a QORA holder and will receive additional QORT, so it's not easy to pre-calculate her balance + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance-(expectedLevel1And2RewardV1*2)); + + } + } + private int getFlags(Repository repository, String name) throws DataException { TestAccount testAccount = Common.getTestAccount(repository, name); diff --git a/src/test/resources/test-chain-v2-block-timestamps.json b/src/test/resources/test-chain-v2-block-timestamps.json index d041463f..e073f1b2 100644 --- a/src/test/resources/test-chain-v2-block-timestamps.json +++ b/src/test/resources/test-chain-v2-block-timestamps.json @@ -19,13 +19,20 @@ { "height": 11, "reward": 10 }, { "height": 21, "reward": 1 } ], - "sharesByLevel": [ + "sharesByLevelV1": [ { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } + ], "qoraHoldersShareByHeight": [ { "height": 1, "share": 0.20 }, { "height": 1000000, "share": 0.01 } @@ -57,6 +64,7 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, + "sharesByLevelV2Height": 999999, "rewardShareLimitTimestamp": 9999999999999, "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, diff --git a/src/test/resources/test-chain-v2-disable-reference.json b/src/test/resources/test-chain-v2-disable-reference.json index 8a4a58cc..a151c5c1 100644 --- a/src/test/resources/test-chain-v2-disable-reference.json +++ b/src/test/resources/test-chain-v2-disable-reference.json @@ -23,13 +23,20 @@ { "height": 11, "reward": 10 }, { "height": 21, "reward": 1 } ], - "sharesByLevel": [ + "sharesByLevelV1": [ { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } + ], "qoraHoldersShareByHeight": [ { "height": 1, "share": 0.20 }, { "height": 1000000, "share": 0.01 } @@ -60,6 +67,7 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, + "sharesByLevelV2Height": 999999, "rewardShareLimitTimestamp": 9999999999999, "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, diff --git a/src/test/resources/test-chain-v2-founder-rewards.json b/src/test/resources/test-chain-v2-founder-rewards.json index e8f1e52b..b6db1b0c 100644 --- a/src/test/resources/test-chain-v2-founder-rewards.json +++ b/src/test/resources/test-chain-v2-founder-rewards.json @@ -24,13 +24,20 @@ { "height": 11, "reward": 10 }, { "height": 21, "reward": 1 } ], - "sharesByLevel": [ + "sharesByLevelV1": [ { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } + ], "qoraHoldersShareByHeight": [ { "height": 1, "share": 0.20 }, { "height": 1000000, "share": 0.01 } @@ -61,6 +68,7 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, + "sharesByLevelV2Height": 999999, "rewardShareLimitTimestamp": 9999999999999, "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, diff --git a/src/test/resources/test-chain-v2-leftover-reward.json b/src/test/resources/test-chain-v2-leftover-reward.json index 233f6aa0..730cf35e 100644 --- a/src/test/resources/test-chain-v2-leftover-reward.json +++ b/src/test/resources/test-chain-v2-leftover-reward.json @@ -24,13 +24,20 @@ { "height": 11, "reward": 10 }, { "height": 21, "reward": 1 } ], - "sharesByLevel": [ + "sharesByLevelV1": [ { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } + ], "qoraHoldersShareByHeight": [ { "height": 1, "share": 0.20 }, { "height": 1000000, "share": 0.01 } @@ -61,6 +68,7 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, + "sharesByLevelV2Height": 999999, "rewardShareLimitTimestamp": 9999999999999, "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, diff --git a/src/test/resources/test-chain-v2-minting.json b/src/test/resources/test-chain-v2-minting.json index b1a25e86..2080beb7 100644 --- a/src/test/resources/test-chain-v2-minting.json +++ b/src/test/resources/test-chain-v2-minting.json @@ -24,13 +24,20 @@ { "height": 11, "reward": 10 }, { "height": 21, "reward": 1 } ], - "sharesByLevel": [ + "sharesByLevelV1": [ { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } + ], "qoraHoldersShareByHeight": [ { "height": 1, "share": 0.20 }, { "height": 1000000, "share": 0.01 } @@ -61,6 +68,7 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, + "sharesByLevelV2Height": 999999, "rewardShareLimitTimestamp": 9999999999999, "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, diff --git a/src/test/resources/test-chain-v2-no-sig-agg.json b/src/test/resources/test-chain-v2-no-sig-agg.json index 71e1cc3d..b671d8f5 100644 --- a/src/test/resources/test-chain-v2-no-sig-agg.json +++ b/src/test/resources/test-chain-v2-no-sig-agg.json @@ -20,12 +20,19 @@ { "height": 11, "reward": 10 }, { "height": 21, "reward": 1 } ], - "sharesByLevel": [ - { "levels": [ 1, 2 ], "share": 0.05 }, - { "levels": [ 3, 4 ], "share": 0.10 }, - { "levels": [ 5, 6 ], "share": 0.15 }, - { "levels": [ 7, 8 ], "share": 0.20 }, - { "levels": [ 9, 10 ], "share": 0.25 } + "sharesByLevelV1": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } + ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } ], "qoraHoldersShare": 0.20, "qoraPerQortReward": 250, @@ -52,6 +59,7 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, + "sharesByLevelV2Height": 999999, "rewardShareLimitTimestamp": 9999999999999, "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, diff --git a/src/test/resources/test-chain-v2-qora-holder-extremes.json b/src/test/resources/test-chain-v2-qora-holder-extremes.json index a3bd1921..d90635f8 100644 --- a/src/test/resources/test-chain-v2-qora-holder-extremes.json +++ b/src/test/resources/test-chain-v2-qora-holder-extremes.json @@ -24,16 +24,23 @@ { "height": 11, "reward": 10 }, { "height": 21, "reward": 1 } ], - "sharesByLevel": [ + "sharesByLevelV1": [ { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } + ], "qoraHoldersShareByHeight": [ { "height": 1, "share": 0.20 }, - { "height": 5, "share": 0.01 } + { "height": 1000, "share": 0.01 } ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, @@ -60,7 +67,8 @@ "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, - "shareBinFix": 999999, + "shareBinFix": 0, + "sharesByLevelV2Height": 1000, "rewardShareLimitTimestamp": 9999999999999, "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, @@ -93,7 +101,10 @@ { "type": "ACCOUNT_FLAGS", "target": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "andMask": -1, "orMask": 1, "xorMask": 0 }, { "type": "REWARD_SHARE", "minterPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "rewardSharePublicKey": "7PpfnvLSG7y4HPh8hE7KoqAjLCkv7Ui6xw4mKAkbZtox", "sharePercent": 100 }, - { "type": "ACCOUNT_LEVEL", "target": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "level": 8 } + { "type": "ACCOUNT_LEVEL", "target": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "level": 1 }, + { "type": "ACCOUNT_LEVEL", "target": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "level": 2 }, + { "type": "ACCOUNT_LEVEL", "target": "QixPbJUwsaHsVEofJdozU9zgVqkK6aYhrK", "level": 1 }, + { "type": "ACCOUNT_LEVEL", "target": "QaUpHNhT3Ygx6avRiKobuLdusppR5biXjL", "level": 1 } ] } } diff --git a/src/test/resources/test-chain-v2-qora-holder-reduction.json b/src/test/resources/test-chain-v2-qora-holder-reduction.json new file mode 100644 index 00000000..75858057 --- /dev/null +++ b/src/test/resources/test-chain-v2-qora-holder-reduction.json @@ -0,0 +1,107 @@ +{ + "isTestChain": true, + "blockTimestampMargin": 500, + "transactionExpiryPeriod": 86400000, + "maxBlockSize": 2097152, + "maxBytesPerUnitFee": 1024, + "unitFee": "0.1", + "nameRegistrationUnitFees": [ + { "timestamp": 1645372800000, "fee": "5" } + ], + "requireGroupForApproval": false, + "minAccountLevelToRewardShare": 5, + "maxRewardSharesPerFounderMintingAccount": 6, + "maxRewardSharesByTimestamp": [ + { "timestamp": 0, "maxShares": 6 }, + { "timestamp": 9999999999999, "maxShares": 3 } + ], + "founderEffectiveMintingLevel": 10, + "onlineAccountSignaturesMinLifetime": 3600000, + "onlineAccountSignaturesMaxLifetime": 86400000, + "onlineAccountsModulusV2Timestamp": 9999999999999, + "rewardsByHeight": [ + { "height": 1, "reward": 100 }, + { "height": 11, "reward": 10 }, + { "height": 21, "reward": 1 } + ], + "sharesByLevelV1": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } + ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } + ], + "qoraHoldersShareByHeight": [ + { "height": 1, "share": 0.20 }, + { "height": 5, "share": 0.01 } + ], + "qoraPerQortReward": 250, + "minAccountsToActivateShareBin": 30, + "shareBinActivationMinLevel": 7, + "blocksNeededByLevel": [ 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 ], + "blockTimingsByHeight": [ + { "height": 1, "target": 60000, "deviation": 30000, "power": 0.2 } + ], + "ciyamAtSettings": { + "feePerStep": "0.0001", + "maxStepsPerRound": 500, + "stepsPerFunctionCall": 10, + "minutesPerBlock": 1 + }, + "featureTriggers": { + "messageHeight": 0, + "atHeight": 0, + "assetsTimestamp": 0, + "votingTimestamp": 0, + "arbitraryTimestamp": 0, + "powfixTimestamp": 0, + "qortalTimestamp": 0, + "newAssetPricingTimestamp": 0, + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, + "sharesByLevelV2Height": 5, + "rewardShareLimitTimestamp": 9999999999999, + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0, + "transactionV6Timestamp": 0, + "disableReferenceTimestamp": 9999999999999, + "aggregateSignatureTimestamp": 0 + }, + "genesisInfo": { + "version": 4, + "timestamp": 0, + "transactions": [ + { "type": "ISSUE_ASSET", "assetName": "QORT", "description": "QORT native coin", "data": "", "quantity": 0, "isDivisible": true, "fee": 0 }, + { "type": "ISSUE_ASSET", "assetName": "Legacy-QORA", "description": "Representative legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true }, + { "type": "ISSUE_ASSET", "assetName": "QORT-from-QORA", "description": "QORT gained from holding legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true }, + + { "type": "GENESIS", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "amount": "1000000000" }, + { "type": "GENESIS", "recipient": "QixPbJUwsaHsVEofJdozU9zgVqkK6aYhrK", "amount": "1000000" }, + { "type": "GENESIS", "recipient": "QaUpHNhT3Ygx6avRiKobuLdusppR5biXjL", "amount": "1000000" }, + { "type": "GENESIS", "recipient": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "amount": "1000000" }, + + { "type": "GENESIS", "recipient": "QaUpHNhT3Ygx6avRiKobuLdusppR5biXjL", "amount": "637557960.49687541", "assetId": 1 }, + { "type": "GENESIS", "recipient": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "amount": "0.666", "assetId": 1 }, + + { "type": "CREATE_GROUP", "creatorPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "groupName": "dev-group", "description": "developer group", "isOpen": false, "approvalThreshold": "PCT100", "minimumBlockDelay": 0, "maximumBlockDelay": 1440 }, + + { "type": "ISSUE_ASSET", "issuerPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "assetName": "TEST", "description": "test asset", "data": "", "quantity": 1000000, "isDivisible": true, "fee": 0 }, + { "type": "ISSUE_ASSET", "issuerPublicKey": "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry", "assetName": "OTHER", "description": "other test asset", "data": "", "quantity": 1000000, "isDivisible": true, "fee": 0 }, + { "type": "ISSUE_ASSET", "issuerPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "assetName": "GOLD", "description": "gold test asset", "data": "", "quantity": 1000000, "isDivisible": true, "fee": 0 }, + + { "type": "ACCOUNT_FLAGS", "target": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "andMask": -1, "orMask": 1, "xorMask": 0 }, + { "type": "REWARD_SHARE", "minterPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "rewardSharePublicKey": "7PpfnvLSG7y4HPh8hE7KoqAjLCkv7Ui6xw4mKAkbZtox", "sharePercent": 100 }, + + { "type": "ACCOUNT_LEVEL", "target": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "level": 8 } + ] + } +} diff --git a/src/test/resources/test-chain-v2-qora-holder.json b/src/test/resources/test-chain-v2-qora-holder.json index 76d10b0d..14712dae 100644 --- a/src/test/resources/test-chain-v2-qora-holder.json +++ b/src/test/resources/test-chain-v2-qora-holder.json @@ -24,13 +24,20 @@ { "height": 11, "reward": 10 }, { "height": 21, "reward": 1 } ], - "sharesByLevel": [ + "sharesByLevelV1": [ { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } + ], "qoraHoldersShareByHeight": [ { "height": 1, "share": 0.20 }, { "height": 1000000, "share": 0.01 } @@ -61,6 +68,7 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, + "sharesByLevelV2Height": 1000, "rewardShareLimitTimestamp": 9999999999999, "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, diff --git a/src/test/resources/test-chain-v2-reward-levels.json b/src/test/resources/test-chain-v2-reward-levels.json index 5434345d..31390285 100644 --- a/src/test/resources/test-chain-v2-reward-levels.json +++ b/src/test/resources/test-chain-v2-reward-levels.json @@ -24,16 +24,23 @@ { "height": 11, "reward": 10 }, { "height": 21, "reward": 1 } ], - "sharesByLevel": [ + "sharesByLevelV1": [ { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } + ], "qoraHoldersShareByHeight": [ { "height": 1, "share": 0.20 }, - { "height": 1000000, "share": 0.01 } + { "height": 1000, "share": 0.01 } ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 1, @@ -61,6 +68,7 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 6, + "sharesByLevelV2Height": 1000, "rewardShareLimitTimestamp": 9999999999999, "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, diff --git a/src/test/resources/test-chain-v2-reward-scaling.json b/src/test/resources/test-chain-v2-reward-scaling.json index 97736e7e..b02207a6 100644 --- a/src/test/resources/test-chain-v2-reward-scaling.json +++ b/src/test/resources/test-chain-v2-reward-scaling.json @@ -24,13 +24,20 @@ { "height": 11, "reward": 10 }, { "height": 21, "reward": 1 } ], - "sharesByLevel": [ + "sharesByLevelV1": [ { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } + ], "qoraHoldersShareByHeight": [ { "height": 1, "share": 0.20 }, { "height": 1000000, "share": 0.01 } @@ -61,6 +68,7 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, + "sharesByLevelV2Height": 999999, "rewardShareLimitTimestamp": 9999999999999, "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, diff --git a/src/test/resources/test-chain-v2-reward-shares.json b/src/test/resources/test-chain-v2-reward-shares.json index 74ce003c..dd86e03d 100644 --- a/src/test/resources/test-chain-v2-reward-shares.json +++ b/src/test/resources/test-chain-v2-reward-shares.json @@ -23,13 +23,20 @@ { "height": 11, "reward": 10 }, { "height": 21, "reward": 1 } ], - "sharesByLevel": [ + "sharesByLevelV1": [ { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } + ], "qoraHoldersShareByHeight": [ { "height": 1, "share": 0.20 }, { "height": 1000000, "share": 0.01 } @@ -60,6 +67,7 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, + "sharesByLevelV2Height": 999999, "rewardShareLimitTimestamp": 0, "calcChainWeightTimestamp": 0, "newConsensusTimestamp": 0, diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index 138dd5d9..7f58fc11 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -24,13 +24,20 @@ { "height": 11, "reward": 10 }, { "height": 21, "reward": 1 } ], - "sharesByLevel": [ + "sharesByLevelV1": [ { "id": 1, "levels": [ 1, 2 ], "share": 0.05 }, { "id": 2, "levels": [ 3, 4 ], "share": 0.10 }, { "id": 3, "levels": [ 5, 6 ], "share": 0.15 }, { "id": 4, "levels": [ 7, 8 ], "share": 0.20 }, { "id": 5, "levels": [ 9, 10 ], "share": 0.25 } ], + "sharesByLevelV2": [ + { "id": 1, "levels": [ 1, 2 ], "share": 0.06 }, + { "id": 2, "levels": [ 3, 4 ], "share": 0.13 }, + { "id": 3, "levels": [ 5, 6 ], "share": 0.19 }, + { "id": 4, "levels": [ 7, 8 ], "share": 0.26 }, + { "id": 5, "levels": [ 9, 10 ], "share": 0.32 } + ], "qoraHoldersShareByHeight": [ { "height": 1, "share": 0.20 }, { "height": 1000000, "share": 0.01 } @@ -61,6 +68,7 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, + "sharesByLevelV2Height": 999999, "rewardShareLimitTimestamp": 9999999999999, "calcChainWeightTimestamp": 0, "transactionV5Timestamp": 0, diff --git a/src/test/resources/test-settings-v2-qora-holder-reduction.json b/src/test/resources/test-settings-v2-qora-holder-reduction.json new file mode 100644 index 00000000..a489cc12 --- /dev/null +++ b/src/test/resources/test-settings-v2-qora-holder-reduction.json @@ -0,0 +1,11 @@ +{ + "repositoryPath": "testdb", + "restrictedApi": false, + "blockchainConfig": "src/test/resources/test-chain-v2-qora-holder-reduction.json", + "exportPath": "qortal-backup-test", + "bootstrap": false, + "wipeUnconfirmedOnStart": false, + "testNtpOffset": 0, + "minPeers": 0, + "pruneBlockLimit": 100 +} From 6cfd85bdce21305669d368233cb649b88d7deebc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 2 Sep 2022 18:05:12 +0100 Subject: [PATCH 16/83] Skip over ARRR orders in /refundAll and /redeemAll, as ARRR support hasn't been added for these yet. This should fix 500 error which could prevent forced refund attempts for LTC and other chains. It wouldn't have affected any normal trade-bot refund functionality. --- .../qortal/api/resource/CrossChainHtlcResource.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java index fbcde1a6..05921ab1 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java @@ -9,6 +9,7 @@ import io.swagger.v3.oas.annotations.tags.Tag; import java.math.BigDecimal; import java.util.List; +import java.util.Objects; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; @@ -284,6 +285,12 @@ public class CrossChainHtlcResource { continue; } + Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain(); + if (Objects.equals(bitcoiny.getCurrencyCode(), "ARRR")) { + LOGGER.info("Skipping AT {} because ARRR is currently unsupported", atAddress); + continue; + } + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); if (crossChainTradeData == null) { LOGGER.info("Couldn't find crosschain trade data for AT {}", atAddress); @@ -532,6 +539,11 @@ public class CrossChainHtlcResource { try { // Determine foreign blockchain receive address for refund Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain(); + if (Objects.equals(bitcoiny.getCurrencyCode(), "ARRR")) { + LOGGER.info("Skipping AT {} because ARRR is currently unsupported", atAddress); + continue; + } + String receivingAddress = bitcoiny.getUnusedReceiveAddress(tradeBotData.getForeignKey()); LOGGER.info("Attempting to refund P2SH balance associated with AT {}...", atAddress); From ad4308afdfa30d2098aa25eac96bdf355a2e2ee8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 2 Sep 2022 19:12:36 +0100 Subject: [PATCH 17/83] Added POST /crosschain/htlc/importarchivedtrades API endpoint. This imports all trades from TradeBotStatesArchive.json into the repository. --- .../api/resource/CrossChainHtlcResource.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java index 05921ab1..5c77f212 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java @@ -7,9 +7,11 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; +import java.io.IOException; import java.math.BigDecimal; import java.util.List; import java.util.Objects; +import java.util.concurrent.locks.ReentrantLock; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; @@ -22,6 +24,7 @@ import org.bitcoinj.core.*; import org.bitcoinj.script.Script; import org.qortal.api.*; import org.qortal.api.model.CrossChainBitcoinyHTLCStatus; +import org.qortal.controller.Controller; import org.qortal.crosschain.*; import org.qortal.crypto.Crypto; import org.qortal.data.at.ATData; @@ -662,6 +665,48 @@ public class CrossChainHtlcResource { return false; } + @POST + @Path("/importarchivedtrades") + @Operation( + summary = "Imports archived trades from TradeBotStatesArchive.json", + description = "This can be used to recover trades that exist in the archive only, which may be needed if a
" + + "problem occurred during the proof-of-work computation stage of a buy request.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public boolean importArchivedTrades(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + + blockchainLock.lockInterruptibly(); + + try { + repository.importDataFromFile("qortal-backup/TradeBotStatesArchive.json"); + repository.saveChanges(); + + return true; + + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e); + + } finally { + blockchainLock.unlock(); + } + } catch (InterruptedException e) { + // We couldn't lock blockchain to perform import + return false; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { return (lockTimeA - tradeTimeout * 60) * 1000L; } From 8cca6db31637dcf43cbcf524949f629c3b35b6a6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 3 Sep 2022 16:30:03 +0100 Subject: [PATCH 18/83] Use block's online accounts timestamp (instead of main timestamp) for the mempow hard fork. This ensures that we cleanly switch from the old to new online account format, even if a block is minted retroactively. --- src/main/java/org/qortal/block/Block.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index c81eef8a..b60e1acb 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -404,7 +404,7 @@ public class Block { byte[] onlineAccountsSignatures = Qortal25519Extras.aggregateSignatures(signaturesToAggregate); // Add nonces to the end of the online accounts signatures if mempow is active - if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + if (onlineAccountsTimestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { try { // Create ordered list of nonce values List nonces = new ArrayList<>(); @@ -1038,7 +1038,7 @@ public class Block { final int signaturesLength = Transformer.SIGNATURE_LENGTH; final int noncesLength = onlineRewardShares.size() * Transformer.INT_LENGTH; - if (this.blockData.getTimestamp() >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + if (this.blockData.getOnlineAccountsTimestamp() >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { // We expect nonces to be appended to the online accounts signatures if (this.blockData.getOnlineAccountsSignatures().length != signaturesLength + noncesLength) return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED; @@ -1054,7 +1054,7 @@ public class Block { byte[] encodedOnlineAccountSignatures = this.blockData.getOnlineAccountsSignatures(); // Split online account signatures into signature(s) + nonces, then validate the nonces - if (this.blockData.getTimestamp() >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + if (this.blockData.getOnlineAccountsTimestamp() >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { byte[] extractedSignatures = BlockTransformer.extract(encodedOnlineAccountSignatures, 0, signaturesLength); byte[] extractedNonces = BlockTransformer.extract(encodedOnlineAccountSignatures, signaturesLength, onlineRewardShares.size() * Transformer.INT_LENGTH); encodedOnlineAccountSignatures = extractedSignatures; From 8879ec5bb46226075701ed13737b25e29e4de835 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 3 Sep 2022 17:11:13 +0100 Subject: [PATCH 19/83] Avoid computing the proof of work for each online account more than once. --- .../qortal/controller/OnlineAccountsManager.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 05f353e8..a384dd01 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -510,10 +510,26 @@ public class OnlineAccountsManager { byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); List ourOnlineAccounts = new ArrayList<>(); + int remaining = mintingAccounts.size(); for (MintingAccountData mintingAccountData : mintingAccounts) { + remaining--; byte[] privateKey = mintingAccountData.getPrivateKey(); byte[] publicKey = Crypto.toPublicKey(privateKey); + // We don't want to compute the online account nonce and signature again if it already exists + Set onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountsTimestamp, k -> ConcurrentHashMap.newKeySet()); + boolean alreadyExists = onlineAccounts.stream().anyMatch(a -> Arrays.equals(a.getPublicKey(), publicKey)); + if (alreadyExists) { + if (remaining > 0) { + // Move on to next account + continue; + } + else { + // Everything exists, so return true + return true; + } + } + // Generate bytes for mempow byte[] mempowBytes; try { From 23423102e70561471cd42db4ac035624d33f0bd0 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Sep 2022 11:51:24 +0100 Subject: [PATCH 20/83] Use onlineAccountTimestamp for all mempow hard fork related code in OnlineAccountsManager too. --- src/main/java/org/qortal/block/Block.java | 2 +- .../qortal/controller/OnlineAccountsManager.java | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index b60e1acb..0d2deb45 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1076,7 +1076,7 @@ public class Block { // Validate the rest for (OnlineAccountData onlineAccount : onlineAccounts) - if (!OnlineAccountsManager.getInstance().verifyMemoryPoW(onlineAccount, this.blockData.getTimestamp())) + if (!OnlineAccountsManager.getInstance().verifyMemoryPoW(onlineAccount)) return ValidationResult.ONLINE_ACCOUNT_NONCE_INCORRECT; } diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index a384dd01..cf309b23 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -305,8 +305,8 @@ public class OnlineAccountsManager { } // Validate mempow if feature trigger is active - if (now >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { - if (!getInstance().verifyMemoryPoW(onlineAccountData, now)) { + if (onlineAccountTimestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + if (!getInstance().verifyMemoryPoW(onlineAccountData)) { LOGGER.trace(() -> String.format("Rejecting online reward-share for account %s due to invalid PoW nonce", mintingAccount.getAddress())); return false; } @@ -542,7 +542,7 @@ public class OnlineAccountsManager { // Compute nonce Integer nonce; - if (isMemoryPoWActive(NTP.getTime())) { + if (isMemoryPoWActive(onlineAccountsTimestamp)) { try { nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp); if (nonce == null) { @@ -565,7 +565,7 @@ public class OnlineAccountsManager { OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce); // Make sure to verify before adding - if (verifyMemoryPoW(ourOnlineAccountData, NTP.getTime())) { + if (verifyMemoryPoW(ourOnlineAccountData)) { ourOnlineAccounts.add(ourOnlineAccountData); } } @@ -615,7 +615,7 @@ public class OnlineAccountsManager { } private Integer computeMemoryPoW(byte[] bytes, byte[] publicKey, long onlineAccountsTimestamp) throws TimeoutException { - if (!isMemoryPoWActive(NTP.getTime())) { + if (!isMemoryPoWActive(onlineAccountsTimestamp)) { LOGGER.info("Mempow start timestamp not yet reached, and onlineAccountsMemPoWEnabled not enabled in settings"); return null; } @@ -641,8 +641,8 @@ public class OnlineAccountsManager { return nonce; } - public boolean verifyMemoryPoW(OnlineAccountData onlineAccountData, Long timestamp) { - if (!isMemoryPoWActive(timestamp)) { + public boolean verifyMemoryPoW(OnlineAccountData onlineAccountData) { + if (!isMemoryPoWActive(onlineAccountData.getTimestamp())) { // Not active yet, so treat it as valid return true; } From 2a0d5746e61b83000d9523da4b4df25ae75908f1 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Sep 2022 13:19:32 +0100 Subject: [PATCH 21/83] Only compute "next" online account signature if mempow hard fork is active. This minimizes the amount of differences in the first phase of the mempow rollout. --- .../org/qortal/controller/OnlineAccountsManager.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index cf309b23..871dd3b7 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -462,12 +462,14 @@ public class OnlineAccountsManager { return; } - // 'next' timestamp (prioritize this as it's the most important) + // 'next' timestamp (prioritize this as it's the most important, if mempow active) final long nextOnlineAccountsTimestamp = toOnlineAccountTimestamp(now) + getOnlineTimestampModulus(); - boolean success = computeOurAccountsForTimestamp(nextOnlineAccountsTimestamp); - if (!success) { - // We didn't compute the required nonce value(s), and so can't proceed until they have been retried - return; + if (nextOnlineAccountsTimestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + boolean success = computeOurAccountsForTimestamp(nextOnlineAccountsTimestamp); + if (!success) { + // We didn't compute the required nonce value(s), and so can't proceed until they have been retried + return; + } } // 'current' timestamp From 82edc4d9f3dd3ecda1c2b50c0eb9e31fef06c71a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 7 Sep 2022 18:26:30 +0100 Subject: [PATCH 22/83] OnlineAccountsV3Message MIN_PEER_VERSION set to 3.5.1 --- .../org/qortal/network/message/OnlineAccountsV3Message.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java index cdd52939..0c5f6730 100644 --- a/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java +++ b/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java @@ -20,7 +20,7 @@ import java.util.Map; */ public class OnlineAccountsV3Message extends Message { - public static final long MIN_PEER_VERSION = 0x300050000L; // 3.5.0 + public static final long MIN_PEER_VERSION = 0x300050001L; // 3.5.1 private List onlineAccounts; From ba4eeed3581686ef5f3261053607dbfe12994c2a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 7 Sep 2022 18:42:24 +0100 Subject: [PATCH 23/83] Modified GET /arbitrary/resources endpoint (and underlying db queries) to allow filtering names by a list, e.g. "followedNames" or "blockedNames". --- .../qortal/api/resource/ArbitraryResource.java | 17 +++++++++++++++-- .../qortal/repository/ArbitraryRepository.java | 2 +- .../hsqldb/HSQLDBArbitraryRepository.java | 15 +++++++++++---- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index dad941e6..978183c0 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -15,6 +15,7 @@ import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.servlet.ServletContext; @@ -44,6 +45,7 @@ import org.qortal.data.arbitrary.*; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; +import org.qortal.list.ResourceListManager; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -91,6 +93,7 @@ public class ArbitraryResource { @Parameter(ref = "limit") @QueryParam("limit") Integer limit, @Parameter(ref = "offset") @QueryParam("offset") Integer offset, @Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse, + @Parameter(description = "Filter names by list") @QueryParam("namefilter") String nameFilter, @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus, @Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata) { @@ -107,8 +110,18 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "identifier cannot be specified when requesting a default resource"); } + // Load filter from list if needed + List names = null; + if (nameFilter != null) { + names = ResourceListManager.getInstance().getStringsInList(nameFilter); + if (names.isEmpty()) { + // List doesn't exist or is empty - so there will be no matches + return new ArrayList<>(); + } + } + List resources = repository.getArbitraryRepository() - .getArbitraryResources(service, identifier, null, defaultRes, limit, offset, reverse); + .getArbitraryResources(service, identifier, names, defaultRes, limit, offset, reverse); if (resources == null) { return new ArrayList<>(); @@ -216,7 +229,7 @@ public class ArbitraryResource { String name = creatorName.name; if (name != null) { List resources = repository.getArbitraryRepository() - .getArbitraryResources(service, identifier, name, defaultRes, null, null, reverse); + .getArbitraryResources(service, identifier, Arrays.asList(name), defaultRes, null, null, reverse); if (includeStatus != null && includeStatus) { resources = this.addStatusToResources(resources); diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 7a31f40e..75fb0509 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -24,7 +24,7 @@ public interface ArbitraryRepository { public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException; - public List getArbitraryResources(Service service, String identifier, String name, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException; + public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException; public List searchArbitraryResources(Service service, String query, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 2c88b089..c21dd038 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -301,7 +301,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } @Override - public List getArbitraryResources(Service service, String identifier, String name, + public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); @@ -325,9 +325,16 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { bindParams.add(identifier); } - if (name != null) { - sql.append(" AND name = ?"); - bindParams.add(name); + if (names != null && !names.isEmpty()) { + sql.append(" AND name IN (?"); + bindParams.add(names.get(0)); + + for (int i = 1; i < names.size(); ++i) { + sql.append(", ?"); + bindParams.add(names.get(i)); + } + + sql.append(")"); } sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); From 667530e20278880beb21798e745f9b813f6c22d5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 9 Sep 2022 17:12:44 +0100 Subject: [PATCH 24/83] Remove DogecoinACCTv2 as it is no longer being used, and is unsafe. --- .../tradebot/DogecoinACCTv2TradeBot.java | 884 ------------------ .../qortal/controller/tradebot/TradeBot.java | 1 - .../org/qortal/crosschain/DogecoinACCTv2.java | 861 ----------------- .../crosschain/SupportedBlockchain.java | 1 - 4 files changed, 1747 deletions(-) delete mode 100644 src/main/java/org/qortal/controller/tradebot/DogecoinACCTv2TradeBot.java delete mode 100644 src/main/java/org/qortal/crosschain/DogecoinACCTv2.java diff --git a/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv2TradeBot.java b/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv2TradeBot.java deleted file mode 100644 index 96dfd1b1..00000000 --- a/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv2TradeBot.java +++ /dev/null @@ -1,884 +0,0 @@ -package org.qortal.controller.tradebot; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bitcoinj.core.*; -import org.bitcoinj.script.Script.ScriptType; -import org.qortal.account.PrivateKeyAccount; -import org.qortal.account.PublicKeyAccount; -import org.qortal.api.model.crosschain.TradeBotCreateRequest; -import org.qortal.asset.Asset; -import org.qortal.crosschain.*; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.crosschain.TradeBotData; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.transaction.DeployAtTransaction; -import org.qortal.transaction.MessageTransaction; -import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.transform.TransformationException; -import org.qortal.transform.transaction.DeployAtTransactionTransformer; -import org.qortal.utils.Base58; -import org.qortal.utils.NTP; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.toMap; - -/** - * Performing cross-chain trading steps on behalf of user. - *

- * We deal with three different independent state-spaces here: - *

    - *
  • Qortal blockchain
    • - *
  • Foreign blockchain
    • - *
  • Trade-bot entries
    • - *
- */ -public class DogecoinACCTv2TradeBot implements AcctTradeBot { - - private static final Logger LOGGER = LogManager.getLogger(DogecoinACCTv2TradeBot.class); - - public enum State implements TradeBot.StateNameAndValueSupplier { - BOB_WAITING_FOR_AT_CONFIRM(10, false, false), - BOB_WAITING_FOR_MESSAGE(15, true, true), - BOB_WAITING_FOR_AT_REDEEM(25, true, true), - BOB_DONE(30, false, false), - BOB_REFUNDED(35, false, false), - - ALICE_WAITING_FOR_AT_LOCK(85, true, true), - ALICE_DONE(95, false, false), - ALICE_REFUNDING_A(105, true, true), - ALICE_REFUNDED(110, false, false); - - private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); - - public final int value; - public final boolean requiresAtData; - public final boolean requiresTradeData; - - State(int value, boolean requiresAtData, boolean requiresTradeData) { - this.value = value; - this.requiresAtData = requiresAtData; - this.requiresTradeData = requiresTradeData; - } - - public static State valueOf(int value) { - return map.get(value); - } - - @Override - public String getState() { - return this.name(); - } - - @Override - public int getStateValue() { - return this.value; - } - } - - /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ - private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms - - private static DogecoinACCTv2TradeBot instance; - - private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream() - .map(State::name) - .collect(Collectors.toUnmodifiableList()); - - private DogecoinACCTv2TradeBot() { - } - - public static synchronized DogecoinACCTv2TradeBot getInstance() { - if (instance == null) - instance = new DogecoinACCTv2TradeBot(); - - return instance; - } - - @Override - public List getEndStates() { - return this.endStates; - } - - /** - * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for DOGE. - *

- * Generates: - *

    - *
  • new 'trade' private key
    • - *
- * Derives: - *
    - *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • - *
  • 'foreign' (as in Dogecoin) public key, public key hash
    • - *
- * A Qortal AT is then constructed including the following as constants in the 'data segment': - *
    - *
  • 'native'/Qortal 'trade' address - used as a MESSAGE contact
    • - *
  • 'foreign'/Dogecoin public key hash - used by Alice's P2SH scripts to allow redeem
    • - *
  • QORT amount on offer by Bob
    • - *
  • DOGE amount expected in return by Bob (from Alice)
    • - *
  • trading timeout, in case things go wrong and everyone needs to refund
    • - *
- * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. - *

- * Trade-bot will wait for Bob's AT to be deployed before taking next step. - *

- * @param repository - * @param tradeBotCreateRequest - * @return raw, unsigned DEPLOY_AT transaction - * @throws DataException - */ - public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { - byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); - - byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); - byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); - String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); - - byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); - byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); - - // Convert Dogecoin receiving address into public key hash (we only support P2PKH at this time) - Address dogecoinReceivingAddress; - try { - dogecoinReceivingAddress = Address.fromString(Dogecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); - } catch (AddressFormatException e) { - throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress); - } - if (dogecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) - throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress); - - byte[] dogecoinReceivingAccountInfo = dogecoinReceivingAddress.getHash(); - - PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); - - // Deploy AT - long timestamp = NTP.getTime(); - byte[] reference = creator.getLastReference(); - long fee = 0L; - byte[] signature = null; - BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); - - String name = "QORT/DOGE ACCT"; - String description = "QORT/DOGE cross-chain trade"; - String aTType = "ACCT"; - String tags = "ACCT QORT DOGE"; - byte[] creationBytes = DogecoinACCTv2.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount, - tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); - long amount = tradeBotCreateRequest.fundingQortAmount; - - DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); - - DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - DeployAtTransaction.ensureATAddress(deployAtTransactionData); - String atAddress = deployAtTransactionData.getAtAddress(); - - TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, DogecoinACCTv2.NAME, - State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, - creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, - tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, - null, null, - SupportedBlockchain.DOGECOIN.name(), - tradeForeignPublicKey, tradeForeignPublicKeyHash, - tradeBotCreateRequest.foreignAmount, null, null, null, dogecoinReceivingAccountInfo); - - TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); - - // Attempt to backup the trade bot data - TradeBot.backupTradeBotData(repository, null); - - // Return to user for signing and broadcast as we don't have their Qortal private key - try { - return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); - } catch (TransformationException e) { - throw new DataException("Failed to transform DEPLOY_AT transaction?", e); - } - } - - /** - * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching DOGE to an existing offer. - *

- * Requires a chosen trade offer from Bob, passed by crossChainTradeData - * and access to a Dogecoin wallet via xprv58. - *

- * The crossChainTradeData contains the current trade offer state - * as extracted from the AT's data segment. - *

- * Access to a funded wallet is via a Dogecoin BIP32 hierarchical deterministic key, - * passed via xprv58. - * This key will be stored in your node's database - * to allow trade-bot to create/fund the necessary P2SH transactions! - * However, due to the nature of BIP32 keys, it is possible to give the trade-bot - * only a subset of wallet access (see BIP32 for more details). - *

- * As an example, the xprv58 can be extract from a legacy, password-less - * Electrum wallet by going to the console tab and entering:
- * wallet.keystore.xprv
- * which should result in a base58 string starting with either 'xprv' (for Dogecoin main-net) - * or 'tprv' for (Dogecoin test-net). - *

- * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. - *

- * If sufficient funds are available, this method will actually fund the P2SH-A - * with the Dogecoin amount expected by 'Bob'. - *

- * If the Dogecoin transaction is successfully broadcast to the network then - * we also send a MESSAGE to Bob's trade-bot to let them know. - *

- * The trade-bot entry is saved to the repository and the cross-chain trading process commences. - *

- * @param repository - * @param crossChainTradeData chosen trade OFFER that Alice wants to match - * @param xprv58 funded wallet xprv in base58 - * @return true if P2SH-A funding transaction successfully broadcast to Dogecoin network, false otherwise - * @throws DataException - */ - public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { - byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); - byte[] secretA = TradeBot.generateSecret(); - byte[] hashOfSecretA = Crypto.hash160(secretA); - - byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); - byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); - String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); - - byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); - byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); - byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH - - // We need to generate lockTime-A: add tradeTimeout to now - long now = NTP.getTime(); - int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); - - TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, DogecoinACCTv2.NAME, - State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value, - receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, - tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, - secretA, hashOfSecretA, - SupportedBlockchain.DOGECOIN.name(), - tradeForeignPublicKey, tradeForeignPublicKeyHash, - crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); - - // Attempt to backup the trade bot data - // Include tradeBotData as an additional parameter, since it's not in the repository yet - TradeBot.backupTradeBotData(repository, Arrays.asList(tradeBotData)); - - // Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount - long p2shFee; - try { - p2shFee = Dogecoin.getInstance().getP2shFee(now); - } catch (ForeignBlockchainException e) { - LOGGER.debug("Couldn't estimate Dogecoin fees?"); - return ResponseResult.NETWORK_ISSUE; - } - - // Fee for redeem/refund is subtracted from P2SH-A balance. - // Do not include fee for funding transaction as this is covered by buildSpend() - long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/; - - // P2SH-A to be funded - byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); - String p2shAddress = Dogecoin.getInstance().deriveP2shAddress(redeemScriptBytes); - - // Build transaction for funding P2SH-A - Transaction p2shFundingTransaction = Dogecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); - if (p2shFundingTransaction == null) { - LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); - return ResponseResult.BALANCE_ISSUE; - } - - try { - Dogecoin.getInstance().broadcastTransaction(p2shFundingTransaction); - } catch (ForeignBlockchainException e) { - // We couldn't fund P2SH-A at this time - LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); - return ResponseResult.NETWORK_ISSUE; - } - - // Attempt to send MESSAGE to Bob's Qortal trade address - byte[] messageData = DogecoinACCTv2.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); - String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; - - boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); - if (!isMessageAlreadySent) { - PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); - - messageTransaction.computeNonce(); - messageTransaction.sign(sender); - - // reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = messageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) { - LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); - return ResponseResult.NETWORK_ISSUE; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress)); - - return ResponseResult.OK; - } - - @Override - public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { - State tradeBotState = State.valueOf(tradeBotData.getStateValue()); - if (tradeBotState == null) - return true; - - // If the AT doesn't exist then we might as well let the user tidy up - if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) - return true; - - switch (tradeBotState) { - case BOB_WAITING_FOR_AT_CONFIRM: - case ALICE_DONE: - case BOB_DONE: - case ALICE_REFUNDED: - case BOB_REFUNDED: - return true; - - default: - return false; - } - } - - @Override - public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { - State tradeBotState = State.valueOf(tradeBotData.getStateValue()); - if (tradeBotState == null) { - LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); - return; - } - - ATData atData = null; - CrossChainTradeData tradeData = null; - - if (tradeBotState.requiresAtData) { - // Attempt to fetch AT data - atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); - if (atData == null) { - LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); - return; - } - - if (tradeBotState.requiresTradeData) { - tradeData = DogecoinACCTv2.getInstance().populateTradeData(repository, atData); - if (tradeData == null) { - LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); - return; - } - } - } - - switch (tradeBotState) { - case BOB_WAITING_FOR_AT_CONFIRM: - handleBobWaitingForAtConfirm(repository, tradeBotData); - break; - - case BOB_WAITING_FOR_MESSAGE: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_WAITING_FOR_AT_LOCK: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); - break; - - case BOB_WAITING_FOR_AT_REDEEM: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_DONE: - case BOB_DONE: - break; - - case ALICE_REFUNDING_A: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_REFUNDED: - case BOB_REFUNDED: - break; - } - } - - /** - * Trade-bot is waiting for Bob's AT to deploy. - *

- * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. - */ - private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { - if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { - if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) - return; - - // We've waited ages for AT to be confirmed into a block but something has gone awry. - // After this long we assume transaction loss so give up with trade-bot entry too. - tradeBotData.setState(State.BOB_REFUNDED.name()); - tradeBotData.setStateValue(State.BOB_REFUNDED.value); - tradeBotData.setTimestamp(NTP.getTime()); - // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() - repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); - repository.saveChanges(); - - LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); - TradeBot.notifyStateChange(tradeBotData); - return; - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, - () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); - } - - /** - * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. - *

- * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, - * in which case trade-bot is done with this specific trade and finalizes on refunded state. - *

- * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. - *

- * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. - *

- * Assuming P2SH-A has at least expected Dogecoin balance, - * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. - *

- * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. - *

- * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to - * extract secret-A needed to redeem Alice's P2SH. - * @throws ForeignBlockchainException - */ - private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // If AT has finished then Bob likely cancelled his trade offer - if (atData.getIsFinished()) { - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, - () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); - return; - } - - Dogecoin dogecoin = Dogecoin.getInstance(); - - String address = tradeBotData.getTradeNativeAddress(); - List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); - - for (MessageTransactionData messageTransactionData : messageTransactionsData) { - if (messageTransactionData.isText()) - continue; - - // We're expecting: HASH160(secret-A), Alice's Dogecoin pubkeyhash and lockTime-A - byte[] messageData = messageTransactionData.getData(); - DogecoinACCTv2.OfferMessageData offerMessageData = DogecoinACCTv2.extractOfferMessageData(messageData); - if (offerMessageData == null) - continue; - - byte[] aliceForeignPublicKeyHash = offerMessageData.partnerDogecoinPKH; - byte[] hashOfSecretA = offerMessageData.hashOfSecretA; - int lockTimeA = (int) offerMessageData.lockTimeA; - long messageTimestamp = messageTransactionData.getTimestamp(); - int refundTimeout = DogecoinACCTv2.calcRefundTimeout(messageTimestamp, lockTimeA); - - // Determine P2SH-A address and confirm funded - byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); - String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); - - long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); - final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee; - - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // There might be another MESSAGE from someone else with an actually funded P2SH-A... - continue; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // We've already redeemed this? - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, - () -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... - continue; - - case FUNDED: - // Fall-through out of switch... - break; - } - - // Good to go - send MESSAGE to AT - - String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); - - // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume - byte[] outgoingMessageData = DogecoinACCTv2.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); - String messageRecipient = tradeBotData.getAtAddress(); - - boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); - if (!isMessageAlreadySent) { - PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); - - outgoingMessageTransaction.computeNonce(); - outgoingMessageTransaction.sign(sender); - - // reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) { - LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); - return; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, - () -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress)); - - return; - } - } - - /** - * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. - *

- * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow - * this process has taken so long that we've reached P2SH-A's locktime, or that someone else - * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. - *

- * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. - *

- * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice. - *

- * In revealing a valid secret-A, Bob can then redeem the DOGE funds from P2SH-A. - *

- * @throws ForeignBlockchainException - */ - private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) - return; - - Dogecoin dogecoin = Dogecoin.getInstance(); - int lockTimeA = tradeBotData.getLockTimeA(); - - // Refund P2SH-A if we've passed lockTime-A - if (NTP.getTime() >= lockTimeA * 1000L) { - byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); - String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); - - long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); - long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - case FUNDED: - break; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // Already redeemed? - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, - () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); - return; - - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> atData.getIsFinished() - ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) - : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); - - return; - } - - // We're waiting for AT to be in TRADE mode - if (crossChainTradeData.mode != AcctMode.TRADING) - return; - - // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above - - // Find our MESSAGE to AT from previous state - List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), - crossChainTradeData.qortalCreatorTradeAddress, null, null, null); - if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { - LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); - return; - } - - long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); - int refundTimeout = DogecoinACCTv2.calcRefundTimeout(recipientMessageTimestamp, lockTimeA); - - // Our calculated refundTimeout should match AT's refundTimeout - if (refundTimeout != crossChainTradeData.refundTimeout) { - LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout)); - // We'll eventually refund - return; - } - - // We're good to redeem AT - - // Send 'redeem' MESSAGE to AT using both secret - byte[] secretA = tradeBotData.getSecret(); - String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH - byte[] messageData = DogecoinACCTv2.buildRedeemMessage(secretA, qortalReceivingAddress); - String messageRecipient = tradeBotData.getAtAddress(); - - boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); - if (!isMessageAlreadySent) { - PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); - - messageTransaction.computeNonce(); - messageTransaction.sign(sender); - - // Reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = messageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) { - LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); - return; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("Redeeming AT %s. Funds should arrive at %s", - tradeBotData.getAtAddress(), qortalReceivingAddress)); - } - - /** - * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the DOGE funds from P2SH-A. - *

- * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, - * trade-bot is done with this specific trade and finalizes in refunded state. - *

- * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the DOGE funds from P2SH-A - * to Bob's 'foreign'/Dogecoin trade legacy-format address, as derived from trade private key. - *

- * (This could potentially be 'improved' to send DOGE to any address of Bob's choosing by changing the transaction output). - *

- * If trade-bot successfully broadcasts the transaction, then this specific trade is done. - * @throws ForeignBlockchainException - */ - private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // AT should be 'finished' once Alice has redeemed QORT funds - if (!atData.getIsFinished()) - // Not finished yet - return; - - // If AT is REFUNDED or CANCELLED then something has gone wrong - if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) { - // Alice hasn't redeemed the QORT, so there is no point in trying to redeem the DOGE - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, - () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); - - return; - } - - byte[] secretA = DogecoinACCTv2.getInstance().findSecretA(repository, crossChainTradeData); - if (secretA == null) { - LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); - return; - } - - // Use secret-A to redeem P2SH-A - - Dogecoin dogecoin = Dogecoin.getInstance(); - - byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); - int lockTimeA = crossChainTradeData.lockTimeA; - byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); - String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); - - // Fee for redeem/refund is subtracted from P2SH-A balance. - long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); - long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund - return; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // Double-check that we have redeemed P2SH-A... - break; - - case REFUND_IN_PROGRESS: - case REFUNDED: - // Wait for AT to auto-refund - return; - - case FUNDED: { - Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); - ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); - List fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA); - - Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(dogecoin.getNetworkParameters(), redeemAmount, redeemKey, - fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); - - dogecoin.broadcastTransaction(p2shRedeemTransaction); - break; - } - } - - String receivingAddress = dogecoin.pkhToAddress(receivingAccountInfo); - - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, - () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); - } - - /** - * Trade-bot is attempting to refund P2SH-A. - * @throws ForeignBlockchainException - */ - private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - int lockTimeA = tradeBotData.getLockTimeA(); - - // We can't refund P2SH-A until lockTime-A has passed - if (NTP.getTime() <= lockTimeA * 1000L) - return; - - Dogecoin dogecoin = Dogecoin.getInstance(); - - // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) - int medianBlockTime = dogecoin.getMedianBlockTime(); - if (medianBlockTime <= lockTimeA) - return; - - byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); - String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA); - - // Fee for redeem/refund is subtracted from P2SH-A balance. - long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp); - long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // Still waiting for P2SH-A to be funded... - return; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // Too late! - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("P2SH-A %s already spent!", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - break; - - case FUNDED:{ - Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); - ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); - List fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA); - - // Determine receive address for refund - String receiveAddress = dogecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); - Address receiving = Address.fromString(dogecoin.getNetworkParameters(), receiveAddress); - - Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(dogecoin.getNetworkParameters(), refundAmount, refundKey, - fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); - - dogecoin.broadcastTransaction(p2shRefundTransaction); - break; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, - () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); - } - - /** - * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. - *

- * Will automatically update trade-bot state to ALICE_REFUNDING_A or ALICE_DONE as necessary. - * - * @throws DataException - * @throws ForeignBlockchainException - */ - private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // This is OK - if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) - return false; - - boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); - - if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING) - if (isAtLockedToUs) { - // AT is trading with us - OK - return false; - } else { - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress)); - - return true; - } - - if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { - // We've redeemed already? - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); - } else { - // Any other state is not good, so start defensive refund - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); - } - - return true; - } - - private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { - return (lockTimeA - tradeTimeout * 60) * 1000L; - } - -} diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index 05ba8f1b..147481f4 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -99,7 +99,6 @@ public class TradeBot implements Listener { acctTradeBotSuppliers.put(LitecoinACCTv2.class, LitecoinACCTv2TradeBot::getInstance); acctTradeBotSuppliers.put(LitecoinACCTv3.class, LitecoinACCTv3TradeBot::getInstance); acctTradeBotSuppliers.put(DogecoinACCTv1.class, DogecoinACCTv1TradeBot::getInstance); - acctTradeBotSuppliers.put(DogecoinACCTv2.class, DogecoinACCTv2TradeBot::getInstance); acctTradeBotSuppliers.put(DogecoinACCTv3.class, DogecoinACCTv3TradeBot::getInstance); acctTradeBotSuppliers.put(DigibyteACCTv3.class, DigibyteACCTv3TradeBot::getInstance); acctTradeBotSuppliers.put(RavencoinACCTv3.class, RavencoinACCTv3TradeBot::getInstance); diff --git a/src/main/java/org/qortal/crosschain/DogecoinACCTv2.java b/src/main/java/org/qortal/crosschain/DogecoinACCTv2.java deleted file mode 100644 index c4b0edb3..00000000 --- a/src/main/java/org/qortal/crosschain/DogecoinACCTv2.java +++ /dev/null @@ -1,861 +0,0 @@ -package org.qortal.crosschain; - -import com.google.common.hash.HashCode; -import com.google.common.primitives.Bytes; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.ciyam.at.*; -import org.qortal.account.Account; -import org.qortal.asset.Asset; -import org.qortal.at.QortalFunctionCode; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.at.ATStateData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.utils.Base58; -import org.qortal.utils.BitTwiddling; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; - -import static org.ciyam.at.OpCode.calcOffset; - -/** - * Cross-chain trade AT - * - *

- *

    - *
  • Bob generates Dogecoin & Qortal 'trade' keys - *
      - *
    • private key required to sign P2SH redeem tx
      • - *
    • private key could be used to create 'secret' (e.g. double-SHA256)
      • - *
    • encrypted private key could be stored in Qortal AT for access by Bob from any node
      • - *
    - *
    • - *
  • Bob deploys Qortal AT - *
      - *
    - *
    • - *
  • Alice finds Qortal AT and wants to trade - *
      - *
    • Alice generates Dogecoin & Qortal 'trade' keys
      • - *
    • Alice funds Dogecoin P2SH-A
      • - *
    • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: - *
        - *
      • hash-of-secret-A
        • - *
      • her 'trade' Dogecoin PKH
        • - *
      - *
      • - *
    - *
    • - *
  • Bob receives "offer" MESSAGE - *
      - *
    • Checks Alice's P2SH-A
      • - *
    • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: - *
        - *
      • Alice's trade Qortal address
        • - *
      • Alice's trade Dogecoin PKH
        • - *
      • hash-of-secret-A
        • - *
      - *
      • - *
    - *
    • - *
  • Alice checks Qortal AT to confirm it's locked to her - *
      - *
    • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: - *
        - *
      • secret-A
        • - *
      • Qortal receiving address of her chosing
        • - *
      - *
      • - *
    • AT's QORT funds are sent to Qortal receiving address
      • - *
    - *
    • - *
  • Bob checks AT, extracts secret-A - *
      - *
    • Bob redeems P2SH-A using his Dogecoin trade key and secret-A
      • - *
    • P2SH-A DOGE funds end up at Dogecoin address determined by redeem transaction output(s)
      • - *
    - *
    • - *
- */ -public class DogecoinACCTv2 implements ACCT { - - private static final Logger LOGGER = LogManager.getLogger(DogecoinACCTv2.class); - - public static final String NAME = DogecoinACCTv2.class.getSimpleName(); - public static final byte[] CODE_BYTES_HASH = HashCode.fromString("6fff38d6eeb06568a9c879c5628527730319844aa0de53f5f4ffab5506efe885").asBytes(); // SHA256 of AT code bytes - - public static final int SECRET_LENGTH = 32; - - /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ - private static final int MODE_VALUE_OFFSET = 61; - /** Byte offset into AT state data where 'mode' variable (long) is stored. */ - public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); - - public static class OfferMessageData { - public byte[] partnerDogecoinPKH; - public byte[] hashOfSecretA; - public long lockTimeA; - } - public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerDogecoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; - public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ - + 24 /*partner's Dogecoin PKH (padded from 20 to 24)*/ - + 8 /*AT trade timeout (minutes)*/ - + 24 /*hash of secret-A (padded from 20 to 24)*/ - + 8 /*lockTimeA*/; - public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret-A*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; - public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; - - private static DogecoinACCTv2 instance; - - private DogecoinACCTv2() { - } - - public static synchronized DogecoinACCTv2 getInstance() { - if (instance == null) - instance = new DogecoinACCTv2(); - - return instance; - } - - @Override - public byte[] getCodeBytesHash() { - return CODE_BYTES_HASH; - } - - @Override - public int getModeByteOffset() { - return MODE_BYTE_OFFSET; - } - - @Override - public ForeignBlockchain getBlockchain() { - return Dogecoin.getInstance(); - } - - /** - * Returns Qortal AT creation bytes for cross-chain trading AT. - *

- * tradeTimeout (minutes) is the time window for the trade partner to send the - * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. - * - * @param creatorTradeAddress AT creator's trade Qortal address - * @param dogecoinPublicKeyHash 20-byte HASH160 of creator's trade Dogecoin public key - * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT - * @param dogecoinAmount how much DOGE the AT creator is expecting to trade - * @param tradeTimeout suggested timeout for entire trade - */ - public static byte[] buildQortalAT(String creatorTradeAddress, byte[] dogecoinPublicKeyHash, long qortAmount, long dogecoinAmount, int tradeTimeout) { - if (dogecoinPublicKeyHash.length != 20) - throw new IllegalArgumentException("Dogecoin public key hash should be 20 bytes"); - - // Labels for data segment addresses - int addrCounter = 0; - - // Constants (with corresponding dataByteBuffer.put*() calls below) - - final int addrCreatorTradeAddress1 = addrCounter++; - final int addrCreatorTradeAddress2 = addrCounter++; - final int addrCreatorTradeAddress3 = addrCounter++; - final int addrCreatorTradeAddress4 = addrCounter++; - - final int addrDogecoinPublicKeyHash = addrCounter; - addrCounter += 4; - - final int addrQortAmount = addrCounter++; - final int addrDogecoinAmount = addrCounter++; - final int addrTradeTimeout = addrCounter++; - - final int addrMessageTxnType = addrCounter++; - final int addrExpectedTradeMessageLength = addrCounter++; - final int addrExpectedRedeemMessageLength = addrCounter++; - - final int addrCreatorAddressPointer = addrCounter++; - final int addrQortalPartnerAddressPointer = addrCounter++; - final int addrMessageSenderPointer = addrCounter++; - - final int addrTradeMessagePartnerDogecoinPKHOffset = addrCounter++; - final int addrPartnerDogecoinPKHPointer = addrCounter++; - final int addrTradeMessageHashOfSecretAOffset = addrCounter++; - final int addrHashOfSecretAPointer = addrCounter++; - - final int addrRedeemMessageReceivingAddressOffset = addrCounter++; - - final int addrMessageDataPointer = addrCounter++; - final int addrMessageDataLength = addrCounter++; - - final int addrPartnerReceivingAddressPointer = addrCounter++; - - final int addrEndOfConstants = addrCounter; - - // Variables - - final int addrCreatorAddress1 = addrCounter++; - final int addrCreatorAddress2 = addrCounter++; - final int addrCreatorAddress3 = addrCounter++; - final int addrCreatorAddress4 = addrCounter++; - - final int addrQortalPartnerAddress1 = addrCounter++; - final int addrQortalPartnerAddress2 = addrCounter++; - final int addrQortalPartnerAddress3 = addrCounter++; - final int addrQortalPartnerAddress4 = addrCounter++; - - final int addrLockTimeA = addrCounter++; - final int addrRefundTimeout = addrCounter++; - final int addrRefundTimestamp = addrCounter++; - final int addrLastTxnTimestamp = addrCounter++; - final int addrBlockTimestamp = addrCounter++; - final int addrTxnType = addrCounter++; - final int addrResult = addrCounter++; - - final int addrMessageSender1 = addrCounter++; - final int addrMessageSender2 = addrCounter++; - final int addrMessageSender3 = addrCounter++; - final int addrMessageSender4 = addrCounter++; - - final int addrMessageLength = addrCounter++; - - final int addrMessageData = addrCounter; - addrCounter += 4; - - final int addrHashOfSecretA = addrCounter; - addrCounter += 4; - - final int addrPartnerDogecoinPKH = addrCounter; - addrCounter += 4; - - final int addrPartnerReceivingAddress = addrCounter; - addrCounter += 4; - - final int addrMode = addrCounter++; - assert addrMode == MODE_VALUE_OFFSET : String.format("addrMode %d does not match MODE_VALUE_OFFSET %d", addrMode, MODE_VALUE_OFFSET); - - // Data segment - ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); - - // AT creator's trade Qortal address, decoded from Base58 - assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; - byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); - dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); - - // Dogecoin public key hash - assert dataByteBuffer.position() == addrDogecoinPublicKeyHash * MachineState.VALUE_SIZE : "addrDogecoinPublicKeyHash incorrect"; - dataByteBuffer.put(Bytes.ensureCapacity(dogecoinPublicKeyHash, 32, 0)); - - // Redeem Qort amount - assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; - dataByteBuffer.putLong(qortAmount); - - // Expected Dogecoin amount - assert dataByteBuffer.position() == addrDogecoinAmount * MachineState.VALUE_SIZE : "addrDogecoinAmount incorrect"; - dataByteBuffer.putLong(dogecoinAmount); - - // Suggested trade timeout (minutes) - assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; - dataByteBuffer.putLong(tradeTimeout); - - // We're only interested in MESSAGE transactions - assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; - dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); - - // Expected length of 'trade' MESSAGE data from AT creator - assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; - dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); - - // Expected length of 'redeem' MESSAGE data from trade partner - assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; - dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); - - // Index into data segment of AT creator's address, used by GET_B_IND - assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; - dataByteBuffer.putLong(addrCreatorAddress1); - - // Index into data segment of partner's Qortal address, used by SET_B_IND - assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; - dataByteBuffer.putLong(addrQortalPartnerAddress1); - - // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND - assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; - dataByteBuffer.putLong(addrMessageSender1); - - // Offset into 'trade' MESSAGE data payload for extracting partner's Dogecoin PKH - assert dataByteBuffer.position() == addrTradeMessagePartnerDogecoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerDogecoinPKHOffset incorrect"; - dataByteBuffer.putLong(32L); - - // Index into data segment of partner's Dogecoin PKH, used by GET_B_IND - assert dataByteBuffer.position() == addrPartnerDogecoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerDogecoinPKHPointer incorrect"; - dataByteBuffer.putLong(addrPartnerDogecoinPKH); - - // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A - assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; - dataByteBuffer.putLong(64L); - - // Index into data segment to hash of secret A, used by GET_B_IND - assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; - dataByteBuffer.putLong(addrHashOfSecretA); - - // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address - assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; - dataByteBuffer.putLong(32L); - - // Source location and length for hashing any passed secret - assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; - dataByteBuffer.putLong(addrMessageData); - assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; - dataByteBuffer.putLong(32L); - - // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND - assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; - dataByteBuffer.putLong(addrPartnerReceivingAddress); - - assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; - - // Code labels - Integer labelRefund = null; - - Integer labelTradeTxnLoop = null; - Integer labelCheckTradeTxn = null; - Integer labelCheckCancelTxn = null; - Integer labelNotTradeNorCancelTxn = null; - Integer labelCheckNonRefundTradeTxn = null; - Integer labelTradeTxnExtract = null; - Integer labelRedeemTxnLoop = null; - Integer labelCheckRedeemTxn = null; - Integer labelCheckRedeemTxnSender = null; - Integer labelPayout = null; - - ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); - - // Two-pass version - for (int pass = 0; pass < 2; ++pass) { - codeByteBuffer.clear(); - - try { - /* Initialization */ - - /* NOP - to ensure DOGECOIN ACCT is unique */ - codeByteBuffer.put(OpCode.NOP.compile()); - - // Use AT creation 'timestamp' as starting point for finding transactions sent to AT - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); - - // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); - - // Set restart position to after this opcode - codeByteBuffer.put(OpCode.SET_PCS.compile()); - - /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ - - /* Transaction processing loop */ - labelTradeTxnLoop = codeByteBuffer.position(); - - /* Sleep until message arrives */ - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.SLEEP_UNTIL_MESSAGE.value, addrLastTxnTimestamp)); - - // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); - // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); - // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction - codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); - // Stop and wait for next block - codeByteBuffer.put(OpCode.STP_IMD.compile()); - - /* Check transaction */ - labelCheckTradeTxn = codeByteBuffer.position(); - - // Update our 'last found transaction's timestamp' using 'timestamp' from transaction - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); - // Extract transaction type (message/payment) from transaction and save type in addrTxnType - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); - // If transaction type is not MESSAGE type then go look for another transaction - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); - - /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ - - // Extract sender address from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); - // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - // Message sender's address matches AT creator's trade address so go process 'trade' message - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); - - /* Checking message sender for possible cancel message */ - labelCheckCancelTxn = codeByteBuffer.position(); - - // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - // Partner address is AT creator's address, so cancel offer and finish. - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); - // We're finished forever (finishing auto-refunds remaining balance to AT creator) - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - - /* Not trade nor cancel message */ - labelNotTradeNorCancelTxn = codeByteBuffer.position(); - - // Loop to find another transaction - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); - - /* Possible switch-to-trade-mode message */ - labelCheckNonRefundTradeTxn = codeByteBuffer.position(); - - // Check 'trade' message we received has expected number of message bytes - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); - // If message length matches, branch to info extraction code - codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); - // Message length didn't match - go back to finding another 'trade' MESSAGE transaction - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); - - /* Extracting info from 'trade' MESSAGE transaction */ - labelTradeTxnExtract = codeByteBuffer.position(); - - // Extract message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); - - // Extract trade partner's Dogecoin public key hash (PKH) from message into B - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerDogecoinPKHOffset)); - // Store partner's Dogecoin PKH (we only really use values from B1-B3) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerDogecoinPKHPointer)); - // Extract AT trade timeout (minutes) (from B4) - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrRefundTimeout)); - - // Grab next 32 bytes - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); - - // Extract hash-of-secret-A (we only really use values from B1-B3) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); - // Extract lockTime-A (from B4) - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); - - // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp - codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); - - /* We are in 'trade mode' */ - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); - - // Set restart position to after this opcode - codeByteBuffer.put(OpCode.SET_PCS.compile()); - - /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ - - // Fetch current block 'timestamp' - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); - // If we're not past refund 'timestamp' then look for next transaction - codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - // We're past refund 'timestamp' so go refund everything back to AT creator - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); - - /* Transaction processing loop */ - labelRedeemTxnLoop = codeByteBuffer.position(); - - /* Sleep until message arrives */ - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.SLEEP_UNTIL_MESSAGE.value, addrLastTxnTimestamp)); - - // Find next transaction to this AT since the last one (if any) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); - // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); - // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction - codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); - // Stop and wait for next block - codeByteBuffer.put(OpCode.STP_IMD.compile()); - - /* Check transaction */ - labelCheckRedeemTxn = codeByteBuffer.position(); - - // Update our 'last found transaction's timestamp' using 'timestamp' from transaction - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); - // Extract transaction type (message/payment) from transaction and save type in addrTxnType - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); - // If transaction type is not MESSAGE type then go look for another transaction - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - - /* Check message payload length */ - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); - // If message length matches, branch to sender checking code - codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); - // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); - - /* Check transaction's sender */ - labelCheckRedeemTxnSender = codeByteBuffer.position(); - - // Extract sender address from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); - // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - - /* Check 'secret-A' in transaction's message */ - - // Extract secret-A from first 32 bytes of message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); - // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); - // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). - // Save the equality result (1 if they match, 0 otherwise) into addrResult. - codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); - // If hashes don't match, addrResult will be zero so go find another transaction - codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); - - /* Success! Pay arranged amount to receiving address */ - labelPayout = codeByteBuffer.position(); - - // Extract Qortal receiving address from next 32 bytes of message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); - // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); - // Pay AT's balance to receiving address - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); - // Set redeemed mode - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); - // We're finished forever (finishing auto-refunds remaining balance to AT creator) - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - - // Fall-through to refunding any remaining balance back to AT creator - - /* Refund balance back to AT creator */ - labelRefund = codeByteBuffer.position(); - - // Set refunded mode - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); - // We're finished forever (finishing auto-refunds remaining balance to AT creator) - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - } catch (CompilationException e) { - throw new IllegalStateException("Unable to compile DOGE-QORT ACCT?", e); - } - } - - codeByteBuffer.flip(); - - byte[] codeBytes = new byte[codeByteBuffer.limit()]; - codeByteBuffer.get(codeBytes); - - assert Arrays.equals(Crypto.digest(codeBytes), DogecoinACCTv2.CODE_BYTES_HASH) - : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); - - final short ciyamAtVersion = 2; - final short numCallStackPages = 0; - final short numUserStackPages = 0; - final long minActivationAmount = 0L; - - return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - */ - @Override - public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { - ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - */ - @Override - public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { - ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { - byte[] addressBytes = new byte[25]; // for general use - String atAddress = atStateData.getATAddress(); - - CrossChainTradeData tradeData = new CrossChainTradeData(); - - tradeData.foreignBlockchain = SupportedBlockchain.DOGECOIN.name(); - tradeData.acctName = NAME; - - tradeData.qortalAtAddress = atAddress; - tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); - tradeData.creationTimestamp = creationTimestamp; - - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); - - byte[] stateData = atStateData.getStateData(); - ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); - dataByteBuffer.position(MachineState.HEADER_LENGTH); - - /* Constants */ - - // Skip creator's trade address - dataByteBuffer.get(addressBytes); - tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); - dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); - - // Creator's Dogecoin/foreign public key hash - tradeData.creatorForeignPKH = new byte[20]; - dataByteBuffer.get(tradeData.creatorForeignPKH); - dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes - - // We don't use secret-B - tradeData.hashOfSecretB = null; - - // Redeem payout - tradeData.qortAmount = dataByteBuffer.getLong(); - - // Expected DOGE amount - tradeData.expectedForeignAmount = dataByteBuffer.getLong(); - - // Trade timeout - tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); - - // Skip MESSAGE transaction type - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip expected 'trade' message length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip expected 'redeem' message length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to creator's address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to partner's Qortal trade address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to message sender - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'trade' message data offset for partner's Dogecoin PKH - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to partner's Dogecoin PKH - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'trade' message data offset for hash-of-secret-A - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to hash-of-secret-A - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'redeem' message data offset for partner's Qortal receiving address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to message data - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip message data length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to partner's receiving address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - /* End of constants / begin variables */ - - // Skip AT creator's address - dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); - - // Partner's trade address (if present) - dataByteBuffer.get(addressBytes); - String qortalRecipient = Base58.encode(addressBytes); - dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); - - // Potential lockTimeA (if in trade mode) - int lockTimeA = (int) dataByteBuffer.getLong(); - - // AT refund timeout (probably only useful for debugging) - int refundTimeout = (int) dataByteBuffer.getLong(); - - // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) - long tradeRefundTimestamp = dataByteBuffer.getLong(); - - // Skip last transaction timestamp - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip block timestamp - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip transaction type - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip temporary result - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip temporary message sender - dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); - - // Skip message length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip temporary message data - dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); - - // Potential hash160 of secret A - byte[] hashOfSecretA = new byte[20]; - dataByteBuffer.get(hashOfSecretA); - dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes - - // Potential partner's Dogecoin PKH - byte[] partnerDogecoinPKH = new byte[20]; - dataByteBuffer.get(partnerDogecoinPKH); - dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerDogecoinPKH.length); // skip to 32 bytes - - // Partner's receiving address (if present) - byte[] partnerReceivingAddress = new byte[25]; - dataByteBuffer.get(partnerReceivingAddress); - dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes - - // Trade AT's 'mode' - long modeValue = dataByteBuffer.getLong(); - AcctMode mode = AcctMode.valueOf((int) (modeValue & 0xffL)); - - /* End of variables */ - - if (mode != null && mode != AcctMode.OFFERING) { - tradeData.mode = mode; - tradeData.refundTimeout = refundTimeout; - tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; - tradeData.qortalPartnerAddress = qortalRecipient; - tradeData.hashOfSecretA = hashOfSecretA; - tradeData.partnerForeignPKH = partnerDogecoinPKH; - tradeData.lockTimeA = lockTimeA; - - if (mode == AcctMode.REDEEMED) - tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); - } else { - tradeData.mode = AcctMode.OFFERING; - } - - tradeData.duplicateDeprecated(); - - return tradeData; - } - - /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ - public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { - byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); - return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); - } - - /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ - public static OfferMessageData extractOfferMessageData(byte[] messageData) { - if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) - return null; - - OfferMessageData offerMessageData = new OfferMessageData(); - offerMessageData.partnerDogecoinPKH = Arrays.copyOfRange(messageData, 0, 20); - offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); - offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); - - return offerMessageData; - } - - /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ - public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int refundTimeout) { - byte[] data = new byte[TRADE_MESSAGE_LENGTH]; - byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); - byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); - byte[] refundTimeoutBytes = BitTwiddling.toBEByteArray((long) refundTimeout); - - System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); - System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); - System.arraycopy(refundTimeoutBytes, 0, data, 56, refundTimeoutBytes.length); - System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); - System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); - - return data; - } - - /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ - @Override - public byte[] buildCancelMessage(String creatorQortalAddress) { - byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; - byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); - - System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); - - return data; - } - - /** Returns 'redeem' MESSAGE payload for trade partner to send to AT. */ - public static byte[] buildRedeemMessage(byte[] secretA, String qortalReceivingAddress) { - byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; - byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); - - System.arraycopy(secretA, 0, data, 0, secretA.length); - System.arraycopy(qortalReceivingAddressBytes, 0, data, 32, qortalReceivingAddressBytes.length); - - return data; - } - - /** Returns refund timeout (minutes) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ - public static int calcRefundTimeout(long offerMessageTimestamp, int lockTimeA) { - // refund should be triggered halfway between offerMessageTimestamp and lockTimeA - return (int) ((lockTimeA - (offerMessageTimestamp / 1000L)) / 2L / 60L); - } - - @Override - public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { - String atAddress = crossChainTradeData.qortalAtAddress; - String redeemerAddress = crossChainTradeData.qortalPartnerAddress; - - // We don't have partner's public key so we check every message to AT - List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); - if (messageTransactionsData == null) - return null; - - // Find 'redeem' message - for (MessageTransactionData messageTransactionData : messageTransactionsData) { - // Check message payload type/encryption - if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) - continue; - - // Check message payload size - byte[] messageData = messageTransactionData.getData(); - if (messageData.length != REDEEM_MESSAGE_LENGTH) - // Wrong payload length - continue; - - // Check sender - if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) - // Wrong sender; - continue; - - // Extract secretA - byte[] secretA = new byte[32]; - System.arraycopy(messageData, 0, secretA, 0, secretA.length); - - byte[] hashOfSecretA = Crypto.hash160(secretA); - if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) - continue; - - return secretA; - } - - return null; - } - -} diff --git a/src/main/java/org/qortal/crosschain/SupportedBlockchain.java b/src/main/java/org/qortal/crosschain/SupportedBlockchain.java index c1f68831..6de8e02d 100644 --- a/src/main/java/org/qortal/crosschain/SupportedBlockchain.java +++ b/src/main/java/org/qortal/crosschain/SupportedBlockchain.java @@ -45,7 +45,6 @@ public enum SupportedBlockchain { DOGECOIN(Arrays.asList( Triple.valueOf(DogecoinACCTv1.NAME, DogecoinACCTv1.CODE_BYTES_HASH, DogecoinACCTv1::getInstance), - Triple.valueOf(DogecoinACCTv2.NAME, DogecoinACCTv2.CODE_BYTES_HASH, DogecoinACCTv2::getInstance), Triple.valueOf(DogecoinACCTv3.NAME, DogecoinACCTv3.CODE_BYTES_HASH, DogecoinACCTv3::getInstance) )) { @Override From c883dd44c8e1efc427067f26317cd1b824326676 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 9 Sep 2022 17:40:10 +0100 Subject: [PATCH 25/83] Remove LitecoinACCTv2 as it is no longer being used, and is unsafe. --- .../tradebot/LitecoinACCTv2TradeBot.java | 885 ------------------ .../qortal/controller/tradebot/TradeBot.java | 1 - .../org/qortal/crosschain/LitecoinACCTv2.java | 854 ----------------- .../crosschain/SupportedBlockchain.java | 1 - 4 files changed, 1741 deletions(-) delete mode 100644 src/main/java/org/qortal/controller/tradebot/LitecoinACCTv2TradeBot.java delete mode 100644 src/main/java/org/qortal/crosschain/LitecoinACCTv2.java diff --git a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv2TradeBot.java b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv2TradeBot.java deleted file mode 100644 index 6261339a..00000000 --- a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv2TradeBot.java +++ /dev/null @@ -1,885 +0,0 @@ -package org.qortal.controller.tradebot; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bitcoinj.core.*; -import org.bitcoinj.script.Script.ScriptType; -import org.qortal.account.PrivateKeyAccount; -import org.qortal.account.PublicKeyAccount; -import org.qortal.api.model.crosschain.TradeBotCreateRequest; -import org.qortal.asset.Asset; -import org.qortal.crosschain.*; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.crosschain.TradeBotData; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.transaction.DeployAtTransaction; -import org.qortal.transaction.MessageTransaction; -import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.transform.TransformationException; -import org.qortal.transform.transaction.DeployAtTransactionTransformer; -import org.qortal.utils.Base58; -import org.qortal.utils.NTP; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.toMap; - -/** - * Performing cross-chain trading steps on behalf of user. - *

- * We deal with three different independent state-spaces here: - *

    - *
  • Qortal blockchain
    • - *
  • Foreign blockchain
    • - *
  • Trade-bot entries
    • - *
- */ -public class LitecoinACCTv2TradeBot implements AcctTradeBot { - - private static final Logger LOGGER = LogManager.getLogger(LitecoinACCTv2TradeBot.class); - - public enum State implements TradeBot.StateNameAndValueSupplier { - BOB_WAITING_FOR_AT_CONFIRM(10, false, false), - BOB_WAITING_FOR_MESSAGE(15, true, true), - BOB_WAITING_FOR_AT_REDEEM(25, true, true), - BOB_DONE(30, false, false), - BOB_REFUNDED(35, false, false), - - ALICE_WAITING_FOR_AT_LOCK(85, true, true), - ALICE_DONE(95, false, false), - ALICE_REFUNDING_A(105, true, true), - ALICE_REFUNDED(110, false, false); - - private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); - - public final int value; - public final boolean requiresAtData; - public final boolean requiresTradeData; - - State(int value, boolean requiresAtData, boolean requiresTradeData) { - this.value = value; - this.requiresAtData = requiresAtData; - this.requiresTradeData = requiresTradeData; - } - - public static State valueOf(int value) { - return map.get(value); - } - - @Override - public String getState() { - return this.name(); - } - - @Override - public int getStateValue() { - return this.value; - } - } - - /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ - private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms - - private static LitecoinACCTv2TradeBot instance; - - private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream() - .map(State::name) - .collect(Collectors.toUnmodifiableList()); - - private LitecoinACCTv2TradeBot() { - } - - public static synchronized LitecoinACCTv2TradeBot getInstance() { - if (instance == null) - instance = new LitecoinACCTv2TradeBot(); - - return instance; - } - - @Override - public List getEndStates() { - return this.endStates; - } - - /** - * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for LTC. - *

- * Generates: - *

    - *
  • new 'trade' private key
    • - *
- * Derives: - *
    - *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • - *
  • 'foreign' (as in Litecoin) public key, public key hash
    • - *
- * A Qortal AT is then constructed including the following as constants in the 'data segment': - *
    - *
  • 'native'/Qortal 'trade' address - used as a MESSAGE contact
    • - *
  • 'foreign'/Litecoin public key hash - used by Alice's P2SH scripts to allow redeem
    • - *
  • QORT amount on offer by Bob
    • - *
  • LTC amount expected in return by Bob (from Alice)
    • - *
  • trading timeout, in case things go wrong and everyone needs to refund
    • - *
- * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. - *

- * Trade-bot will wait for Bob's AT to be deployed before taking next step. - *

- * @param repository - * @param tradeBotCreateRequest - * @return raw, unsigned DEPLOY_AT transaction - * @throws DataException - */ - public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { - byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); - - byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); - byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); - String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); - - byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); - byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); - - // Convert Litecoin receiving address into public key hash (we only support P2PKH at this time) - Address litecoinReceivingAddress; - try { - litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); - } catch (AddressFormatException e) { - throw new DataException("Unsupported Litecoin receiving address: " + tradeBotCreateRequest.receivingAddress); - } - if (litecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) - throw new DataException("Unsupported Litecoin receiving address: " + tradeBotCreateRequest.receivingAddress); - - byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash(); - - PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); - - // Deploy AT - long timestamp = NTP.getTime(); - byte[] reference = creator.getLastReference(); - long fee = 0L; - byte[] signature = null; - BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); - - String name = "QORT/LTC ACCT"; - String description = "QORT/LTC cross-chain trade"; - String aTType = "ACCT"; - String tags = "ACCT QORT LTC"; - byte[] creationBytes = LitecoinACCTv2.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount, - tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); - long amount = tradeBotCreateRequest.fundingQortAmount; - - DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); - - DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - DeployAtTransaction.ensureATAddress(deployAtTransactionData); - String atAddress = deployAtTransactionData.getAtAddress(); - - TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv2.NAME, - State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, - creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, - tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, - null, null, - SupportedBlockchain.LITECOIN.name(), - tradeForeignPublicKey, tradeForeignPublicKeyHash, - tradeBotCreateRequest.foreignAmount, null, null, null, litecoinReceivingAccountInfo); - - TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); - - // Attempt to backup the trade bot data - TradeBot.backupTradeBotData(repository, null); - - // Return to user for signing and broadcast as we don't have their Qortal private key - try { - return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); - } catch (TransformationException e) { - throw new DataException("Failed to transform DEPLOY_AT transaction?", e); - } - } - - /** - * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching LTC to an existing offer. - *

- * Requires a chosen trade offer from Bob, passed by crossChainTradeData - * and access to a Litecoin wallet via xprv58. - *

- * The crossChainTradeData contains the current trade offer state - * as extracted from the AT's data segment. - *

- * Access to a funded wallet is via a Litecoin BIP32 hierarchical deterministic key, - * passed via xprv58. - * This key will be stored in your node's database - * to allow trade-bot to create/fund the necessary P2SH transactions! - * However, due to the nature of BIP32 keys, it is possible to give the trade-bot - * only a subset of wallet access (see BIP32 for more details). - *

- * As an example, the xprv58 can be extract from a legacy, password-less - * Electrum wallet by going to the console tab and entering:
- * wallet.keystore.xprv
- * which should result in a base58 string starting with either 'xprv' (for Litecoin main-net) - * or 'tprv' for (Litecoin test-net). - *

- * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. - *

- * If sufficient funds are available, this method will actually fund the P2SH-A - * with the Litecoin amount expected by 'Bob'. - *

- * If the Litecoin transaction is successfully broadcast to the network then - * we also send a MESSAGE to Bob's trade-bot to let them know. - *

- * The trade-bot entry is saved to the repository and the cross-chain trading process commences. - *

- * @param repository - * @param crossChainTradeData chosen trade OFFER that Alice wants to match - * @param xprv58 funded wallet xprv in base58 - * @return true if P2SH-A funding transaction successfully broadcast to Litecoin network, false otherwise - * @throws DataException - */ - public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { - byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); - byte[] secretA = TradeBot.generateSecret(); - byte[] hashOfSecretA = Crypto.hash160(secretA); - - byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); - byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); - String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); - - byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); - byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); - byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH - - // We need to generate lockTime-A: add tradeTimeout to now - long now = NTP.getTime(); - int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); - - TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv2.NAME, - State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value, - receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, - tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, - secretA, hashOfSecretA, - SupportedBlockchain.LITECOIN.name(), - tradeForeignPublicKey, tradeForeignPublicKeyHash, - crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); - - // Attempt to backup the trade bot data - // Include tradeBotData as an additional parameter, since it's not in the repository yet - TradeBot.backupTradeBotData(repository, Arrays.asList(tradeBotData)); - - // Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount - long p2shFee; - try { - p2shFee = Litecoin.getInstance().getP2shFee(now); - } catch (ForeignBlockchainException e) { - LOGGER.debug("Couldn't estimate Litecoin fees?"); - return ResponseResult.NETWORK_ISSUE; - } - - // Fee for redeem/refund is subtracted from P2SH-A balance. - // Do not include fee for funding transaction as this is covered by buildSpend() - long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/; - - // P2SH-A to be funded - byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); - String p2shAddress = Litecoin.getInstance().deriveP2shAddress(redeemScriptBytes); - - // Build transaction for funding P2SH-A - Transaction p2shFundingTransaction = Litecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); - if (p2shFundingTransaction == null) { - LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); - return ResponseResult.BALANCE_ISSUE; - } - - try { - Litecoin.getInstance().broadcastTransaction(p2shFundingTransaction); - } catch (ForeignBlockchainException e) { - // We couldn't fund P2SH-A at this time - LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); - return ResponseResult.NETWORK_ISSUE; - } - - // Attempt to send MESSAGE to Bob's Qortal trade address - byte[] messageData = LitecoinACCTv2.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); - String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; - - boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); - if (!isMessageAlreadySent) { - PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); - - messageTransaction.computeNonce(); - messageTransaction.sign(sender); - - // reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = messageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) { - LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); - return ResponseResult.NETWORK_ISSUE; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress)); - - return ResponseResult.OK; - } - - @Override - public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { - State tradeBotState = State.valueOf(tradeBotData.getStateValue()); - if (tradeBotState == null) - return true; - - // If the AT doesn't exist then we might as well let the user tidy up - if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) - return true; - - switch (tradeBotState) { - case BOB_WAITING_FOR_AT_CONFIRM: - case ALICE_DONE: - case BOB_DONE: - case ALICE_REFUNDED: - case BOB_REFUNDED: - case ALICE_REFUNDING_A: - return true; - - default: - return false; - } - } - - @Override - public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { - State tradeBotState = State.valueOf(tradeBotData.getStateValue()); - if (tradeBotState == null) { - LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); - return; - } - - ATData atData = null; - CrossChainTradeData tradeData = null; - - if (tradeBotState.requiresAtData) { - // Attempt to fetch AT data - atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); - if (atData == null) { - LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); - return; - } - - if (tradeBotState.requiresTradeData) { - tradeData = LitecoinACCTv2.getInstance().populateTradeData(repository, atData); - if (tradeData == null) { - LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); - return; - } - } - } - - switch (tradeBotState) { - case BOB_WAITING_FOR_AT_CONFIRM: - handleBobWaitingForAtConfirm(repository, tradeBotData); - break; - - case BOB_WAITING_FOR_MESSAGE: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_WAITING_FOR_AT_LOCK: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); - break; - - case BOB_WAITING_FOR_AT_REDEEM: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_DONE: - case BOB_DONE: - break; - - case ALICE_REFUNDING_A: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_REFUNDED: - case BOB_REFUNDED: - break; - } - } - - /** - * Trade-bot is waiting for Bob's AT to deploy. - *

- * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. - */ - private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { - if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { - if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) - return; - - // We've waited ages for AT to be confirmed into a block but something has gone awry. - // After this long we assume transaction loss so give up with trade-bot entry too. - tradeBotData.setState(State.BOB_REFUNDED.name()); - tradeBotData.setStateValue(State.BOB_REFUNDED.value); - tradeBotData.setTimestamp(NTP.getTime()); - // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() - repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); - repository.saveChanges(); - - LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); - TradeBot.notifyStateChange(tradeBotData); - return; - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, - () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); - } - - /** - * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. - *

- * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, - * in which case trade-bot is done with this specific trade and finalizes on refunded state. - *

- * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. - *

- * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. - *

- * Assuming P2SH-A has at least expected Litecoin balance, - * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. - *

- * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. - *

- * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to - * extract secret-A needed to redeem Alice's P2SH. - * @throws ForeignBlockchainException - */ - private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // If AT has finished then Bob likely cancelled his trade offer - if (atData.getIsFinished()) { - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, - () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); - return; - } - - Litecoin litecoin = Litecoin.getInstance(); - - String address = tradeBotData.getTradeNativeAddress(); - List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); - - for (MessageTransactionData messageTransactionData : messageTransactionsData) { - if (messageTransactionData.isText()) - continue; - - // We're expecting: HASH160(secret-A), Alice's Litecoin pubkeyhash and lockTime-A - byte[] messageData = messageTransactionData.getData(); - LitecoinACCTv2.OfferMessageData offerMessageData = LitecoinACCTv2.extractOfferMessageData(messageData); - if (offerMessageData == null) - continue; - - byte[] aliceForeignPublicKeyHash = offerMessageData.partnerLitecoinPKH; - byte[] hashOfSecretA = offerMessageData.hashOfSecretA; - int lockTimeA = (int) offerMessageData.lockTimeA; - long messageTimestamp = messageTransactionData.getTimestamp(); - int refundTimeout = LitecoinACCTv2.calcRefundTimeout(messageTimestamp, lockTimeA); - - // Determine P2SH-A address and confirm funded - byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); - String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); - - long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); - final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee; - - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // There might be another MESSAGE from someone else with an actually funded P2SH-A... - continue; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // We've already redeemed this? - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, - () -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... - continue; - - case FUNDED: - // Fall-through out of switch... - break; - } - - // Good to go - send MESSAGE to AT - - String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); - - // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume - byte[] outgoingMessageData = LitecoinACCTv2.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); - String messageRecipient = tradeBotData.getAtAddress(); - - boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); - if (!isMessageAlreadySent) { - PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); - - outgoingMessageTransaction.computeNonce(); - outgoingMessageTransaction.sign(sender); - - // reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) { - LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); - return; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, - () -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress)); - - return; - } - } - - /** - * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. - *

- * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow - * this process has taken so long that we've reached P2SH-A's locktime, or that someone else - * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. - *

- * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. - *

- * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice. - *

- * In revealing a valid secret-A, Bob can then redeem the LTC funds from P2SH-A. - *

- * @throws ForeignBlockchainException - */ - private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) - return; - - Litecoin litecoin = Litecoin.getInstance(); - int lockTimeA = tradeBotData.getLockTimeA(); - - // Refund P2SH-A if we've passed lockTime-A - if (NTP.getTime() >= lockTimeA * 1000L) { - byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); - String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); - - long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); - long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - case FUNDED: - break; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // Already redeemed? - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, - () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); - return; - - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> atData.getIsFinished() - ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) - : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); - - return; - } - - // We're waiting for AT to be in TRADE mode - if (crossChainTradeData.mode != AcctMode.TRADING) - return; - - // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above - - // Find our MESSAGE to AT from previous state - List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), - crossChainTradeData.qortalCreatorTradeAddress, null, null, null); - if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { - LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); - return; - } - - long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); - int refundTimeout = LitecoinACCTv2.calcRefundTimeout(recipientMessageTimestamp, lockTimeA); - - // Our calculated refundTimeout should match AT's refundTimeout - if (refundTimeout != crossChainTradeData.refundTimeout) { - LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout)); - // We'll eventually refund - return; - } - - // We're good to redeem AT - - // Send 'redeem' MESSAGE to AT using both secret - byte[] secretA = tradeBotData.getSecret(); - String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH - byte[] messageData = LitecoinACCTv2.buildRedeemMessage(secretA, qortalReceivingAddress); - String messageRecipient = tradeBotData.getAtAddress(); - - boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); - if (!isMessageAlreadySent) { - PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); - - messageTransaction.computeNonce(); - messageTransaction.sign(sender); - - // Reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = messageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) { - LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); - return; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("Redeeming AT %s. Funds should arrive at %s", - tradeBotData.getAtAddress(), qortalReceivingAddress)); - } - - /** - * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the LTC funds from P2SH-A. - *

- * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, - * trade-bot is done with this specific trade and finalizes in refunded state. - *

- * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the LTC funds from P2SH-A - * to Bob's 'foreign'/Litecoin trade legacy-format address, as derived from trade private key. - *

- * (This could potentially be 'improved' to send LTC to any address of Bob's choosing by changing the transaction output). - *

- * If trade-bot successfully broadcasts the transaction, then this specific trade is done. - * @throws ForeignBlockchainException - */ - private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // AT should be 'finished' once Alice has redeemed QORT funds - if (!atData.getIsFinished()) - // Not finished yet - return; - - // If AT is REFUNDED or CANCELLED then something has gone wrong - if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) { - // Alice hasn't redeemed the QORT, so there is no point in trying to redeem the LTC - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, - () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); - - return; - } - - byte[] secretA = LitecoinACCTv2.getInstance().findSecretA(repository, crossChainTradeData); - if (secretA == null) { - LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); - return; - } - - // Use secret-A to redeem P2SH-A - - Litecoin litecoin = Litecoin.getInstance(); - - byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); - int lockTimeA = crossChainTradeData.lockTimeA; - byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); - String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); - - // Fee for redeem/refund is subtracted from P2SH-A balance. - long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); - long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund - return; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // Double-check that we have redeemed P2SH-A... - break; - - case REFUND_IN_PROGRESS: - case REFUNDED: - // Wait for AT to auto-refund - return; - - case FUNDED: { - Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); - ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); - List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); - - Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(litecoin.getNetworkParameters(), redeemAmount, redeemKey, - fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); - - litecoin.broadcastTransaction(p2shRedeemTransaction); - break; - } - } - - String receivingAddress = litecoin.pkhToAddress(receivingAccountInfo); - - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, - () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); - } - - /** - * Trade-bot is attempting to refund P2SH-A. - * @throws ForeignBlockchainException - */ - private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - int lockTimeA = tradeBotData.getLockTimeA(); - - // We can't refund P2SH-A until lockTime-A has passed - if (NTP.getTime() <= lockTimeA * 1000L) - return; - - Litecoin litecoin = Litecoin.getInstance(); - - // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) - int medianBlockTime = litecoin.getMedianBlockTime(); - if (medianBlockTime <= lockTimeA) - return; - - byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); - String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); - - // Fee for redeem/refund is subtracted from P2SH-A balance. - long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); - long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // Still waiting for P2SH-A to be funded... - return; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // Too late! - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("P2SH-A %s already spent!", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - break; - - case FUNDED:{ - Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); - ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); - List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); - - // Determine receive address for refund - String receiveAddress = litecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); - Address receiving = Address.fromString(litecoin.getNetworkParameters(), receiveAddress); - - Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(litecoin.getNetworkParameters(), refundAmount, refundKey, - fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); - - litecoin.broadcastTransaction(p2shRefundTransaction); - break; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, - () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); - } - - /** - * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. - *

- * Will automatically update trade-bot state to ALICE_REFUNDING_A or ALICE_DONE as necessary. - * - * @throws DataException - * @throws ForeignBlockchainException - */ - private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // This is OK - if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) - return false; - - boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); - - if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING) - if (isAtLockedToUs) { - // AT is trading with us - OK - return false; - } else { - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress)); - - return true; - } - - if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { - // We've redeemed already? - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); - } else { - // Any other state is not good, so start defensive refund - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); - } - - return true; - } - - private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { - return (lockTimeA - tradeTimeout * 60) * 1000L; - } - -} diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index 147481f4..c7ae1db3 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -96,7 +96,6 @@ public class TradeBot implements Listener { acctTradeBotSuppliers.put(BitcoinACCTv1.class, BitcoinACCTv1TradeBot::getInstance); acctTradeBotSuppliers.put(BitcoinACCTv3.class, BitcoinACCTv3TradeBot::getInstance); acctTradeBotSuppliers.put(LitecoinACCTv1.class, LitecoinACCTv1TradeBot::getInstance); - acctTradeBotSuppliers.put(LitecoinACCTv2.class, LitecoinACCTv2TradeBot::getInstance); acctTradeBotSuppliers.put(LitecoinACCTv3.class, LitecoinACCTv3TradeBot::getInstance); acctTradeBotSuppliers.put(DogecoinACCTv1.class, DogecoinACCTv1TradeBot::getInstance); acctTradeBotSuppliers.put(DogecoinACCTv3.class, DogecoinACCTv3TradeBot::getInstance); diff --git a/src/main/java/org/qortal/crosschain/LitecoinACCTv2.java b/src/main/java/org/qortal/crosschain/LitecoinACCTv2.java deleted file mode 100644 index c5728953..00000000 --- a/src/main/java/org/qortal/crosschain/LitecoinACCTv2.java +++ /dev/null @@ -1,854 +0,0 @@ -package org.qortal.crosschain; - -import com.google.common.hash.HashCode; -import com.google.common.primitives.Bytes; -import org.ciyam.at.*; -import org.qortal.account.Account; -import org.qortal.asset.Asset; -import org.qortal.at.QortalFunctionCode; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.at.ATStateData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.utils.Base58; -import org.qortal.utils.BitTwiddling; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; - -import static org.ciyam.at.OpCode.calcOffset; - -/** - * Cross-chain trade AT - * - *

- *

    - *
  • Bob generates Litecoin & Qortal 'trade' keys - *
      - *
    • private key required to sign P2SH redeem tx
      • - *
    • private key could be used to create 'secret' (e.g. double-SHA256)
      • - *
    • encrypted private key could be stored in Qortal AT for access by Bob from any node
      • - *
    - *
    • - *
  • Bob deploys Qortal AT - *
      - *
    - *
    • - *
  • Alice finds Qortal AT and wants to trade - *
      - *
    • Alice generates Litecoin & Qortal 'trade' keys
      • - *
    • Alice funds Litecoin P2SH-A
      • - *
    • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: - *
        - *
      • hash-of-secret-A
        • - *
      • her 'trade' Litecoin PKH
        • - *
      - *
      • - *
    - *
    • - *
  • Bob receives "offer" MESSAGE - *
      - *
    • Checks Alice's P2SH-A
      • - *
    • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: - *
        - *
      • Alice's trade Qortal address
        • - *
      • Alice's trade Litecoin PKH
        • - *
      • hash-of-secret-A
        • - *
      - *
      • - *
    - *
    • - *
  • Alice checks Qortal AT to confirm it's locked to her - *
      - *
    • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: - *
        - *
      • secret-A
        • - *
      • Qortal receiving address of her chosing
        • - *
      - *
      • - *
    • AT's QORT funds are sent to Qortal receiving address
      • - *
    - *
    • - *
  • Bob checks AT, extracts secret-A - *
      - *
    • Bob redeems P2SH-A using his Litecoin trade key and secret-A
      • - *
    • P2SH-A LTC funds end up at Litecoin address determined by redeem transaction output(s)
      • - *
    - *
    • - *
- */ -public class LitecoinACCTv2 implements ACCT { - - public static final String NAME = LitecoinACCTv2.class.getSimpleName(); - public static final byte[] CODE_BYTES_HASH = HashCode.fromString("d5ea386a41441180c854ca8d7bbc620bfd53a97df2650a2b162b52324caf6e19").asBytes(); // SHA256 of AT code bytes - - public static final int SECRET_LENGTH = 32; - - /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ - private static final int MODE_VALUE_OFFSET = 61; - /** Byte offset into AT state data where 'mode' variable (long) is stored. */ - public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); - - public static class OfferMessageData { - public byte[] partnerLitecoinPKH; - public byte[] hashOfSecretA; - public long lockTimeA; - } - public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerLitecoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; - public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ - + 24 /*partner's Litecoin PKH (padded from 20 to 24)*/ - + 8 /*AT trade timeout (minutes)*/ - + 24 /*hash of secret-A (padded from 20 to 24)*/ - + 8 /*lockTimeA*/; - public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret-A*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; - public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; - - private static LitecoinACCTv2 instance; - - private LitecoinACCTv2() { - } - - public static synchronized LitecoinACCTv2 getInstance() { - if (instance == null) - instance = new LitecoinACCTv2(); - - return instance; - } - - @Override - public byte[] getCodeBytesHash() { - return CODE_BYTES_HASH; - } - - @Override - public int getModeByteOffset() { - return MODE_BYTE_OFFSET; - } - - @Override - public ForeignBlockchain getBlockchain() { - return Litecoin.getInstance(); - } - - /** - * Returns Qortal AT creation bytes for cross-chain trading AT. - *

- * tradeTimeout (minutes) is the time window for the trade partner to send the - * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. - * - * @param creatorTradeAddress AT creator's trade Qortal address - * @param litecoinPublicKeyHash 20-byte HASH160 of creator's trade Litecoin public key - * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT - * @param litecoinAmount how much LTC the AT creator is expecting to trade - * @param tradeTimeout suggested timeout for entire trade - */ - public static byte[] buildQortalAT(String creatorTradeAddress, byte[] litecoinPublicKeyHash, long qortAmount, long litecoinAmount, int tradeTimeout) { - if (litecoinPublicKeyHash.length != 20) - throw new IllegalArgumentException("Litecoin public key hash should be 20 bytes"); - - // Labels for data segment addresses - int addrCounter = 0; - - // Constants (with corresponding dataByteBuffer.put*() calls below) - - final int addrCreatorTradeAddress1 = addrCounter++; - final int addrCreatorTradeAddress2 = addrCounter++; - final int addrCreatorTradeAddress3 = addrCounter++; - final int addrCreatorTradeAddress4 = addrCounter++; - - final int addrLitecoinPublicKeyHash = addrCounter; - addrCounter += 4; - - final int addrQortAmount = addrCounter++; - final int addrLitecoinAmount = addrCounter++; - final int addrTradeTimeout = addrCounter++; - - final int addrMessageTxnType = addrCounter++; - final int addrExpectedTradeMessageLength = addrCounter++; - final int addrExpectedRedeemMessageLength = addrCounter++; - - final int addrCreatorAddressPointer = addrCounter++; - final int addrQortalPartnerAddressPointer = addrCounter++; - final int addrMessageSenderPointer = addrCounter++; - - final int addrTradeMessagePartnerLitecoinPKHOffset = addrCounter++; - final int addrPartnerLitecoinPKHPointer = addrCounter++; - final int addrTradeMessageHashOfSecretAOffset = addrCounter++; - final int addrHashOfSecretAPointer = addrCounter++; - - final int addrRedeemMessageReceivingAddressOffset = addrCounter++; - - final int addrMessageDataPointer = addrCounter++; - final int addrMessageDataLength = addrCounter++; - - final int addrPartnerReceivingAddressPointer = addrCounter++; - - final int addrEndOfConstants = addrCounter; - - // Variables - - final int addrCreatorAddress1 = addrCounter++; - final int addrCreatorAddress2 = addrCounter++; - final int addrCreatorAddress3 = addrCounter++; - final int addrCreatorAddress4 = addrCounter++; - - final int addrQortalPartnerAddress1 = addrCounter++; - final int addrQortalPartnerAddress2 = addrCounter++; - final int addrQortalPartnerAddress3 = addrCounter++; - final int addrQortalPartnerAddress4 = addrCounter++; - - final int addrLockTimeA = addrCounter++; - final int addrRefundTimeout = addrCounter++; - final int addrRefundTimestamp = addrCounter++; - final int addrLastTxnTimestamp = addrCounter++; - final int addrBlockTimestamp = addrCounter++; - final int addrTxnType = addrCounter++; - final int addrResult = addrCounter++; - - final int addrMessageSender1 = addrCounter++; - final int addrMessageSender2 = addrCounter++; - final int addrMessageSender3 = addrCounter++; - final int addrMessageSender4 = addrCounter++; - - final int addrMessageLength = addrCounter++; - - final int addrMessageData = addrCounter; - addrCounter += 4; - - final int addrHashOfSecretA = addrCounter; - addrCounter += 4; - - final int addrPartnerLitecoinPKH = addrCounter; - addrCounter += 4; - - final int addrPartnerReceivingAddress = addrCounter; - addrCounter += 4; - - final int addrMode = addrCounter++; - assert addrMode == MODE_VALUE_OFFSET : String.format("addrMode %d does not match MODE_VALUE_OFFSET %d", addrMode, MODE_VALUE_OFFSET); - - // Data segment - ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); - - // AT creator's trade Qortal address, decoded from Base58 - assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; - byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); - dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); - - // Litecoin public key hash - assert dataByteBuffer.position() == addrLitecoinPublicKeyHash * MachineState.VALUE_SIZE : "addrLitecoinPublicKeyHash incorrect"; - dataByteBuffer.put(Bytes.ensureCapacity(litecoinPublicKeyHash, 32, 0)); - - // Redeem Qort amount - assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; - dataByteBuffer.putLong(qortAmount); - - // Expected Litecoin amount - assert dataByteBuffer.position() == addrLitecoinAmount * MachineState.VALUE_SIZE : "addrLitecoinAmount incorrect"; - dataByteBuffer.putLong(litecoinAmount); - - // Suggested trade timeout (minutes) - assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; - dataByteBuffer.putLong(tradeTimeout); - - // We're only interested in MESSAGE transactions - assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; - dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); - - // Expected length of 'trade' MESSAGE data from AT creator - assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; - dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); - - // Expected length of 'redeem' MESSAGE data from trade partner - assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; - dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); - - // Index into data segment of AT creator's address, used by GET_B_IND - assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; - dataByteBuffer.putLong(addrCreatorAddress1); - - // Index into data segment of partner's Qortal address, used by SET_B_IND - assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; - dataByteBuffer.putLong(addrQortalPartnerAddress1); - - // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND - assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; - dataByteBuffer.putLong(addrMessageSender1); - - // Offset into 'trade' MESSAGE data payload for extracting partner's Litecoin PKH - assert dataByteBuffer.position() == addrTradeMessagePartnerLitecoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerLitecoinPKHOffset incorrect"; - dataByteBuffer.putLong(32L); - - // Index into data segment of partner's Litecoin PKH, used by GET_B_IND - assert dataByteBuffer.position() == addrPartnerLitecoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerLitecoinPKHPointer incorrect"; - dataByteBuffer.putLong(addrPartnerLitecoinPKH); - - // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A - assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; - dataByteBuffer.putLong(64L); - - // Index into data segment to hash of secret A, used by GET_B_IND - assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; - dataByteBuffer.putLong(addrHashOfSecretA); - - // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address - assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; - dataByteBuffer.putLong(32L); - - // Source location and length for hashing any passed secret - assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; - dataByteBuffer.putLong(addrMessageData); - assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; - dataByteBuffer.putLong(32L); - - // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND - assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; - dataByteBuffer.putLong(addrPartnerReceivingAddress); - - assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; - - // Code labels - Integer labelRefund = null; - - Integer labelTradeTxnLoop = null; - Integer labelCheckTradeTxn = null; - Integer labelCheckCancelTxn = null; - Integer labelNotTradeNorCancelTxn = null; - Integer labelCheckNonRefundTradeTxn = null; - Integer labelTradeTxnExtract = null; - Integer labelRedeemTxnLoop = null; - Integer labelCheckRedeemTxn = null; - Integer labelCheckRedeemTxnSender = null; - Integer labelPayout = null; - - ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); - - // Two-pass version - for (int pass = 0; pass < 2; ++pass) { - codeByteBuffer.clear(); - - try { - /* Initialization */ - - // Use AT creation 'timestamp' as starting point for finding transactions sent to AT - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); - - // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); - - // Set restart position to after this opcode - codeByteBuffer.put(OpCode.SET_PCS.compile()); - - /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ - - /* Transaction processing loop */ - labelTradeTxnLoop = codeByteBuffer.position(); - - /* Sleep until message arrives */ - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.SLEEP_UNTIL_MESSAGE.value, addrLastTxnTimestamp)); - - // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); - // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); - // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction - codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); - // Stop and wait for next block - codeByteBuffer.put(OpCode.STP_IMD.compile()); - - /* Check transaction */ - labelCheckTradeTxn = codeByteBuffer.position(); - - // Update our 'last found transaction's timestamp' using 'timestamp' from transaction - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); - // Extract transaction type (message/payment) from transaction and save type in addrTxnType - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); - // If transaction type is not MESSAGE type then go look for another transaction - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); - - /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ - - // Extract sender address from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); - // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - // Message sender's address matches AT creator's trade address so go process 'trade' message - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); - - /* Checking message sender for possible cancel message */ - labelCheckCancelTxn = codeByteBuffer.position(); - - // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - // Partner address is AT creator's address, so cancel offer and finish. - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); - // We're finished forever (finishing auto-refunds remaining balance to AT creator) - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - - /* Not trade nor cancel message */ - labelNotTradeNorCancelTxn = codeByteBuffer.position(); - - // Loop to find another transaction - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); - - /* Possible switch-to-trade-mode message */ - labelCheckNonRefundTradeTxn = codeByteBuffer.position(); - - // Check 'trade' message we received has expected number of message bytes - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); - // If message length matches, branch to info extraction code - codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); - // Message length didn't match - go back to finding another 'trade' MESSAGE transaction - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); - - /* Extracting info from 'trade' MESSAGE transaction */ - labelTradeTxnExtract = codeByteBuffer.position(); - - // Extract message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); - - // Extract trade partner's Litecoin public key hash (PKH) from message into B - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerLitecoinPKHOffset)); - // Store partner's Litecoin PKH (we only really use values from B1-B3) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerLitecoinPKHPointer)); - // Extract AT trade timeout (minutes) (from B4) - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrRefundTimeout)); - - // Grab next 32 bytes - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); - - // Extract hash-of-secret-A (we only really use values from B1-B3) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); - // Extract lockTime-A (from B4) - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); - - // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp - codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); - - /* We are in 'trade mode' */ - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); - - // Set restart position to after this opcode - codeByteBuffer.put(OpCode.SET_PCS.compile()); - - /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ - - // Fetch current block 'timestamp' - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); - // If we're not past refund 'timestamp' then look for next transaction - codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - // We're past refund 'timestamp' so go refund everything back to AT creator - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); - - /* Transaction processing loop */ - labelRedeemTxnLoop = codeByteBuffer.position(); - - /* Sleep until message arrives */ - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.SLEEP_UNTIL_MESSAGE.value, addrLastTxnTimestamp)); - - // Find next transaction to this AT since the last one (if any) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); - // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); - // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction - codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); - // Stop and wait for next block - codeByteBuffer.put(OpCode.STP_IMD.compile()); - - /* Check transaction */ - labelCheckRedeemTxn = codeByteBuffer.position(); - - // Update our 'last found transaction's timestamp' using 'timestamp' from transaction - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); - // Extract transaction type (message/payment) from transaction and save type in addrTxnType - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); - // If transaction type is not MESSAGE type then go look for another transaction - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - - /* Check message payload length */ - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); - // If message length matches, branch to sender checking code - codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); - // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); - - /* Check transaction's sender */ - labelCheckRedeemTxnSender = codeByteBuffer.position(); - - // Extract sender address from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); - // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - - /* Check 'secret-A' in transaction's message */ - - // Extract secret-A from first 32 bytes of message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); - // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); - // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). - // Save the equality result (1 if they match, 0 otherwise) into addrResult. - codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); - // If hashes don't match, addrResult will be zero so go find another transaction - codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); - - /* Success! Pay arranged amount to receiving address */ - labelPayout = codeByteBuffer.position(); - - // Extract Qortal receiving address from next 32 bytes of message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); - // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); - // Pay AT's balance to receiving address - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); - // Set redeemed mode - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); - // We're finished forever (finishing auto-refunds remaining balance to AT creator) - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - - // Fall-through to refunding any remaining balance back to AT creator - - /* Refund balance back to AT creator */ - labelRefund = codeByteBuffer.position(); - - // Set refunded mode - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); - // We're finished forever (finishing auto-refunds remaining balance to AT creator) - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - } catch (CompilationException e) { - throw new IllegalStateException("Unable to compile LTC-QORT ACCT?", e); - } - } - - codeByteBuffer.flip(); - - byte[] codeBytes = new byte[codeByteBuffer.limit()]; - codeByteBuffer.get(codeBytes); - - assert Arrays.equals(Crypto.digest(codeBytes), LitecoinACCTv2.CODE_BYTES_HASH) - : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); - - final short ciyamAtVersion = 2; - final short numCallStackPages = 0; - final short numUserStackPages = 0; - final long minActivationAmount = 0L; - - return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - */ - @Override - public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { - ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - */ - @Override - public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { - ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { - byte[] addressBytes = new byte[25]; // for general use - String atAddress = atStateData.getATAddress(); - - CrossChainTradeData tradeData = new CrossChainTradeData(); - - tradeData.foreignBlockchain = SupportedBlockchain.LITECOIN.name(); - tradeData.acctName = NAME; - - tradeData.qortalAtAddress = atAddress; - tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); - tradeData.creationTimestamp = creationTimestamp; - - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); - - byte[] stateData = atStateData.getStateData(); - ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); - dataByteBuffer.position(MachineState.HEADER_LENGTH); - - /* Constants */ - - // Skip creator's trade address - dataByteBuffer.get(addressBytes); - tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); - dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); - - // Creator's Litecoin/foreign public key hash - tradeData.creatorForeignPKH = new byte[20]; - dataByteBuffer.get(tradeData.creatorForeignPKH); - dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes - - // We don't use secret-B - tradeData.hashOfSecretB = null; - - // Redeem payout - tradeData.qortAmount = dataByteBuffer.getLong(); - - // Expected LTC amount - tradeData.expectedForeignAmount = dataByteBuffer.getLong(); - - // Trade timeout - tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); - - // Skip MESSAGE transaction type - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip expected 'trade' message length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip expected 'redeem' message length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to creator's address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to partner's Qortal trade address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to message sender - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'trade' message data offset for partner's Litecoin PKH - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to partner's Litecoin PKH - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'trade' message data offset for hash-of-secret-A - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to hash-of-secret-A - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'redeem' message data offset for partner's Qortal receiving address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to message data - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip message data length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to partner's receiving address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - /* End of constants / begin variables */ - - // Skip AT creator's address - dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); - - // Partner's trade address (if present) - dataByteBuffer.get(addressBytes); - String qortalRecipient = Base58.encode(addressBytes); - dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); - - // Potential lockTimeA (if in trade mode) - int lockTimeA = (int) dataByteBuffer.getLong(); - - // AT refund timeout (probably only useful for debugging) - int refundTimeout = (int) dataByteBuffer.getLong(); - - // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) - long tradeRefundTimestamp = dataByteBuffer.getLong(); - - // Skip last transaction timestamp - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip block timestamp - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip transaction type - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip temporary result - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip temporary message sender - dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); - - // Skip message length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip temporary message data - dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); - - // Potential hash160 of secret A - byte[] hashOfSecretA = new byte[20]; - dataByteBuffer.get(hashOfSecretA); - dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes - - // Potential partner's Litecoin PKH - byte[] partnerLitecoinPKH = new byte[20]; - dataByteBuffer.get(partnerLitecoinPKH); - dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerLitecoinPKH.length); // skip to 32 bytes - - // Partner's receiving address (if present) - byte[] partnerReceivingAddress = new byte[25]; - dataByteBuffer.get(partnerReceivingAddress); - dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes - - // Trade AT's 'mode' - long modeValue = dataByteBuffer.getLong(); - AcctMode mode = AcctMode.valueOf((int) (modeValue & 0xffL)); - - /* End of variables */ - - if (mode != null && mode != AcctMode.OFFERING) { - tradeData.mode = mode; - tradeData.refundTimeout = refundTimeout; - tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; - tradeData.qortalPartnerAddress = qortalRecipient; - tradeData.hashOfSecretA = hashOfSecretA; - tradeData.partnerForeignPKH = partnerLitecoinPKH; - tradeData.lockTimeA = lockTimeA; - - if (mode == AcctMode.REDEEMED) - tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); - } else { - tradeData.mode = AcctMode.OFFERING; - } - - tradeData.duplicateDeprecated(); - - return tradeData; - } - - /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ - public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { - byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); - return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); - } - - /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ - public static OfferMessageData extractOfferMessageData(byte[] messageData) { - if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) - return null; - - OfferMessageData offerMessageData = new OfferMessageData(); - offerMessageData.partnerLitecoinPKH = Arrays.copyOfRange(messageData, 0, 20); - offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); - offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); - - return offerMessageData; - } - - /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ - public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int refundTimeout) { - byte[] data = new byte[TRADE_MESSAGE_LENGTH]; - byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); - byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); - byte[] refundTimeoutBytes = BitTwiddling.toBEByteArray((long) refundTimeout); - - System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); - System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); - System.arraycopy(refundTimeoutBytes, 0, data, 56, refundTimeoutBytes.length); - System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); - System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); - - return data; - } - - /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ - @Override - public byte[] buildCancelMessage(String creatorQortalAddress) { - byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; - byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); - - System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); - - return data; - } - - /** Returns 'redeem' MESSAGE payload for trade partner to send to AT. */ - public static byte[] buildRedeemMessage(byte[] secretA, String qortalReceivingAddress) { - byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; - byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); - - System.arraycopy(secretA, 0, data, 0, secretA.length); - System.arraycopy(qortalReceivingAddressBytes, 0, data, 32, qortalReceivingAddressBytes.length); - - return data; - } - - /** Returns refund timeout (minutes) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ - public static int calcRefundTimeout(long offerMessageTimestamp, int lockTimeA) { - // refund should be triggered halfway between offerMessageTimestamp and lockTimeA - return (int) ((lockTimeA - (offerMessageTimestamp / 1000L)) / 2L / 60L); - } - - @Override - public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { - String atAddress = crossChainTradeData.qortalAtAddress; - String redeemerAddress = crossChainTradeData.qortalPartnerAddress; - - // We don't have partner's public key so we check every message to AT - List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); - if (messageTransactionsData == null) - return null; - - // Find 'redeem' message - for (MessageTransactionData messageTransactionData : messageTransactionsData) { - // Check message payload type/encryption - if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) - continue; - - // Check message payload size - byte[] messageData = messageTransactionData.getData(); - if (messageData.length != REDEEM_MESSAGE_LENGTH) - // Wrong payload length - continue; - - // Check sender - if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) - // Wrong sender; - continue; - - // Extract secretA - byte[] secretA = new byte[32]; - System.arraycopy(messageData, 0, secretA, 0, secretA.length); - - byte[] hashOfSecretA = Crypto.hash160(secretA); - if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) - continue; - - return secretA; - } - - return null; - } - -} diff --git a/src/main/java/org/qortal/crosschain/SupportedBlockchain.java b/src/main/java/org/qortal/crosschain/SupportedBlockchain.java index 6de8e02d..5ddb6aec 100644 --- a/src/main/java/org/qortal/crosschain/SupportedBlockchain.java +++ b/src/main/java/org/qortal/crosschain/SupportedBlockchain.java @@ -29,7 +29,6 @@ public enum SupportedBlockchain { LITECOIN(Arrays.asList( Triple.valueOf(LitecoinACCTv1.NAME, LitecoinACCTv1.CODE_BYTES_HASH, LitecoinACCTv1::getInstance), - Triple.valueOf(LitecoinACCTv2.NAME, LitecoinACCTv2.CODE_BYTES_HASH, LitecoinACCTv2::getInstance), Triple.valueOf(LitecoinACCTv3.NAME, LitecoinACCTv3.CODE_BYTES_HASH, LitecoinACCTv3::getInstance) )) { @Override From 8ffdc9b3693875a360c6881cab07e7e97173a34d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 9 Sep 2022 18:01:20 +0100 Subject: [PATCH 26/83] POST /crosschain/htlc/importarchivedtrades moved to POST /admin/repository/importarchivedtrades, as this is a repository operation not an HTLC one. --- .../qortal/api/resource/AdminResource.java | 43 +++++++++++++++++++ .../api/resource/CrossChainHtlcResource.java | 42 ------------------ 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index 8a9d8025..9cff1bbb 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -728,6 +728,49 @@ public class AdminResource { } } + @POST + @Path("/repository/importarchivedtrades") + @Operation( + summary = "Imports archived trades from TradeBotStatesArchive.json", + description = "This can be used to recover trades that exist in the archive only, which may be needed if a
" + + "problem occurred during the proof-of-work computation stage of a buy request.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public boolean importArchivedTrades(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + + blockchainLock.lockInterruptibly(); + + try { + repository.importDataFromFile("qortal-backup/TradeBotStatesArchive.json"); + repository.saveChanges(); + + return true; + + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e); + + } finally { + blockchainLock.unlock(); + } + } catch (InterruptedException e) { + // We couldn't lock blockchain to perform import + return false; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST @Path("/apikey/generate") diff --git a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java index 5c77f212..cf098f53 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java @@ -665,48 +665,6 @@ public class CrossChainHtlcResource { return false; } - @POST - @Path("/importarchivedtrades") - @Operation( - summary = "Imports archived trades from TradeBotStatesArchive.json", - description = "This can be used to recover trades that exist in the archive only, which may be needed if a
" + - "problem occurred during the proof-of-work computation stage of a buy request.", - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) - ) - } - ) - @ApiErrors({ApiError.REPOSITORY_ISSUE}) - @SecurityRequirement(name = "apiKey") - public boolean importArchivedTrades(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { - Security.checkApiCallAllowed(request); - - try (final Repository repository = RepositoryManager.getRepository()) { - ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - - blockchainLock.lockInterruptibly(); - - try { - repository.importDataFromFile("qortal-backup/TradeBotStatesArchive.json"); - repository.saveChanges(); - - return true; - - } catch (IOException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e); - - } finally { - blockchainLock.unlock(); - } - } catch (InterruptedException e) { - // We couldn't lock blockchain to perform import - return false; - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { return (lockTimeA - tradeTimeout * 60) * 1000L; } From d4fbc1687ba76b1d5f2f6a30b124f2822a13f037 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 9 Sep 2022 18:45:51 +0100 Subject: [PATCH 27/83] Optionally exclude initial data from all trade websockets, using query string parameter excludeInitialData=true Due to the large amount of data, it can take some time for the request to be processed and data to be transferred. It may make more sense to load the initial state from the standard API, and just use the websockets for updates. --- .../api/websocket/TradeBotWebSocket.java | 29 +++++++++++-------- .../api/websocket/TradeOffersWebSocket.java | 26 ++++++++++------- .../api/websocket/TradePresenceWebSocket.java | 10 +++++-- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java b/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java index 55969c6b..8d7a13cd 100644 --- a/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java @@ -2,10 +2,7 @@ package org.qortal.api.websocket; import java.io.IOException; import java.io.StringWriter; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import org.eclipse.jetty.websocket.api.Session; @@ -85,6 +82,7 @@ public class TradeBotWebSocket extends ApiWebSocket implements Listener { @Override public void onWebSocketConnect(Session session) { Map> queryParams = session.getUpgradeRequest().getParameterMap(); + final boolean excludeInitialData = queryParams.get("excludeInitialData") != null; List foreignBlockchains = queryParams.get("foreignBlockchain"); final String foreignBlockchain = foreignBlockchains == null ? null : foreignBlockchains.get(0); @@ -98,15 +96,22 @@ public class TradeBotWebSocket extends ApiWebSocket implements Listener { // save session's preferred blockchain (if any) sessionBlockchain.put(session, foreignBlockchain); - // Send all known trade-bot entries - try (final Repository repository = RepositoryManager.getRepository()) { - List tradeBotEntries = repository.getCrossChainRepository().getAllTradeBotData(); - // Optional filtering - if (foreignBlockchain != null) - tradeBotEntries = tradeBotEntries.stream() - .filter(tradeBotData -> tradeBotData.getForeignBlockchain().equals(foreignBlockchain)) - .collect(Collectors.toList()); + + // Maybe send all known trade-bot entries + try (final Repository repository = RepositoryManager.getRepository()) { + List tradeBotEntries = new ArrayList<>(); + + // We might need to exclude the initial data from the response + if (!excludeInitialData) { + tradeBotEntries = repository.getCrossChainRepository().getAllTradeBotData(); + + // Optional filtering + if (foreignBlockchain != null) + tradeBotEntries = tradeBotEntries.stream() + .filter(tradeBotData -> tradeBotData.getForeignBlockchain().equals(foreignBlockchain)) + .collect(Collectors.toList()); + } if (!sendEntries(session, tradeBotEntries)) { session.close(4002, "websocket issue"); diff --git a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java index 35fc4691..78c53dc3 100644 --- a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java @@ -173,6 +173,7 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener { public void onWebSocketConnect(Session session) { Map> queryParams = session.getUpgradeRequest().getParameterMap(); final boolean includeHistoric = queryParams.get("includeHistoric") != null; + final boolean excludeInitialData = queryParams.get("excludeInitialData") != null; List foreignBlockchains = queryParams.get("foreignBlockchain"); final String foreignBlockchain = foreignBlockchains == null ? null : foreignBlockchains.get(0); @@ -189,20 +190,23 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener { List crossChainOfferSummaries = new ArrayList<>(); - synchronized (cachedInfoByBlockchain) { - Collection cachedInfos; + // We might need to exclude the initial data from the response + if (!excludeInitialData) { + synchronized (cachedInfoByBlockchain) { + Collection cachedInfos; - if (foreignBlockchain == null) - // No preferred blockchain, so iterate through all of them - cachedInfos = cachedInfoByBlockchain.values(); - else - cachedInfos = Collections.singleton(cachedInfoByBlockchain.computeIfAbsent(foreignBlockchain, k -> new CachedOfferInfo())); + if (foreignBlockchain == null) + // No preferred blockchain, so iterate through all of them + cachedInfos = cachedInfoByBlockchain.values(); + else + cachedInfos = Collections.singleton(cachedInfoByBlockchain.computeIfAbsent(foreignBlockchain, k -> new CachedOfferInfo())); - for (CachedOfferInfo cachedInfo : cachedInfos) { - crossChainOfferSummaries.addAll(cachedInfo.currentSummaries.values()); + for (CachedOfferInfo cachedInfo : cachedInfos) { + crossChainOfferSummaries.addAll(cachedInfo.currentSummaries.values()); - if (includeHistoric) - crossChainOfferSummaries.addAll(cachedInfo.historicSummaries.values()); + if (includeHistoric) + crossChainOfferSummaries.addAll(cachedInfo.historicSummaries.values()); + } } } diff --git a/src/main/java/org/qortal/api/websocket/TradePresenceWebSocket.java b/src/main/java/org/qortal/api/websocket/TradePresenceWebSocket.java index e9558599..ba9a8085 100644 --- a/src/main/java/org/qortal/api/websocket/TradePresenceWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/TradePresenceWebSocket.java @@ -65,11 +65,15 @@ public class TradePresenceWebSocket extends ApiWebSocket implements Listener { @Override public void onWebSocketConnect(Session session) { Map> queryParams = session.getUpgradeRequest().getParameterMap(); + final boolean excludeInitialData = queryParams.get("excludeInitialData") != null; - List tradePresences; + List tradePresences = new ArrayList<>(); - synchronized (currentEntries) { - tradePresences = List.copyOf(currentEntries.values()); + // We might need to exclude the initial data from the response + if (!excludeInitialData) { + synchronized (currentEntries) { + tradePresences = List.copyOf(currentEntries.values()); + } } if (!sendTradePresences(session, tradePresences)) { From 03e36198171c0263614913fc7b2c6e9962595706 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Sep 2022 17:23:34 +0100 Subject: [PATCH 28/83] Revert "Use onlineAccountTimestamp for all mempow hard fork related code in OnlineAccountsManager too." This reverts commit 23423102e70561471cd42db4ac035624d33f0bd0. --- src/main/java/org/qortal/block/Block.java | 2 +- .../qortal/controller/OnlineAccountsManager.java | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 5c482c39..babacb5d 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1079,7 +1079,7 @@ public class Block { // Validate the rest for (OnlineAccountData onlineAccount : onlineAccounts) - if (!OnlineAccountsManager.getInstance().verifyMemoryPoW(onlineAccount)) + if (!OnlineAccountsManager.getInstance().verifyMemoryPoW(onlineAccount, this.blockData.getTimestamp())) return ValidationResult.ONLINE_ACCOUNT_NONCE_INCORRECT; } diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 871dd3b7..013585ee 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -305,8 +305,8 @@ public class OnlineAccountsManager { } // Validate mempow if feature trigger is active - if (onlineAccountTimestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { - if (!getInstance().verifyMemoryPoW(onlineAccountData)) { + if (now >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + if (!getInstance().verifyMemoryPoW(onlineAccountData, now)) { LOGGER.trace(() -> String.format("Rejecting online reward-share for account %s due to invalid PoW nonce", mintingAccount.getAddress())); return false; } @@ -544,7 +544,7 @@ public class OnlineAccountsManager { // Compute nonce Integer nonce; - if (isMemoryPoWActive(onlineAccountsTimestamp)) { + if (isMemoryPoWActive(NTP.getTime())) { try { nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp); if (nonce == null) { @@ -567,7 +567,7 @@ public class OnlineAccountsManager { OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce); // Make sure to verify before adding - if (verifyMemoryPoW(ourOnlineAccountData)) { + if (verifyMemoryPoW(ourOnlineAccountData, NTP.getTime())) { ourOnlineAccounts.add(ourOnlineAccountData); } } @@ -617,7 +617,7 @@ public class OnlineAccountsManager { } private Integer computeMemoryPoW(byte[] bytes, byte[] publicKey, long onlineAccountsTimestamp) throws TimeoutException { - if (!isMemoryPoWActive(onlineAccountsTimestamp)) { + if (!isMemoryPoWActive(NTP.getTime())) { LOGGER.info("Mempow start timestamp not yet reached, and onlineAccountsMemPoWEnabled not enabled in settings"); return null; } @@ -643,8 +643,8 @@ public class OnlineAccountsManager { return nonce; } - public boolean verifyMemoryPoW(OnlineAccountData onlineAccountData) { - if (!isMemoryPoWActive(onlineAccountData.getTimestamp())) { + public boolean verifyMemoryPoW(OnlineAccountData onlineAccountData, Long timestamp) { + if (!isMemoryPoWActive(timestamp)) { // Not active yet, so treat it as valid return true; } From 6003ed3ff7fc51762f9627ce421fdae9f48b762e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Sep 2022 17:23:39 +0100 Subject: [PATCH 29/83] Revert "Use block's online accounts timestamp (instead of main timestamp) for the mempow hard fork." This reverts commit 8cca6db31637dcf43cbcf524949f629c3b35b6a6. --- src/main/java/org/qortal/block/Block.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index babacb5d..293cb3eb 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -407,7 +407,7 @@ public class Block { byte[] onlineAccountsSignatures = Qortal25519Extras.aggregateSignatures(signaturesToAggregate); // Add nonces to the end of the online accounts signatures if mempow is active - if (onlineAccountsTimestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { try { // Create ordered list of nonce values List nonces = new ArrayList<>(); @@ -1041,7 +1041,7 @@ public class Block { final int signaturesLength = Transformer.SIGNATURE_LENGTH; final int noncesLength = onlineRewardShares.size() * Transformer.INT_LENGTH; - if (this.blockData.getOnlineAccountsTimestamp() >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + if (this.blockData.getTimestamp() >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { // We expect nonces to be appended to the online accounts signatures if (this.blockData.getOnlineAccountsSignatures().length != signaturesLength + noncesLength) return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED; @@ -1057,7 +1057,7 @@ public class Block { byte[] encodedOnlineAccountSignatures = this.blockData.getOnlineAccountsSignatures(); // Split online account signatures into signature(s) + nonces, then validate the nonces - if (this.blockData.getOnlineAccountsTimestamp() >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + if (this.blockData.getTimestamp() >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { byte[] extractedSignatures = BlockTransformer.extract(encodedOnlineAccountSignatures, 0, signaturesLength); byte[] extractedNonces = BlockTransformer.extract(encodedOnlineAccountSignatures, signaturesLength, onlineRewardShares.size() * Transformer.INT_LENGTH); encodedOnlineAccountSignatures = extractedSignatures; From 501f66ab00187f9133351a4127ea40f7ed3ac468 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Sep 2022 18:31:01 +0100 Subject: [PATCH 30/83] BlockTransformer updates necessary for mempow online accounts. Using the feature trigger timestamp here should be much less error prone than a whole new block message version. Once mempow has been live for at least 24 hours, the feature trigger can be removed and the code cleaned up, as all online accounts signatures will use the new format from that time onwards (legacy signatures are trimmed after 24 hours). --- src/main/java/org/qortal/data/block/BlockData.java | 8 ++++++++ .../qortal/transform/block/BlockTransformer.java | 14 ++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/data/block/BlockData.java b/src/main/java/org/qortal/data/block/BlockData.java index 61d1a7fb..763bca45 100644 --- a/src/main/java/org/qortal/data/block/BlockData.java +++ b/src/main/java/org/qortal/data/block/BlockData.java @@ -211,6 +211,14 @@ public class BlockData implements Serializable { this.onlineAccountsSignatures = onlineAccountsSignatures; } + public int getOnlineAccountsSignaturesCount() { + if (this.onlineAccountsSignatures != null && this.onlineAccountsSignatures.length > 0) { + // Blocks use a single online accounts signature, so there is no need for this to be dynamic + return 1; + } + return 0; + } + public boolean isTrimmed() { long onlineAccountSignaturesTrimmedTimestamp = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime(); long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime(); diff --git a/src/main/java/org/qortal/transform/block/BlockTransformer.java b/src/main/java/org/qortal/transform/block/BlockTransformer.java index 48e79699..9e02a6f5 100644 --- a/src/main/java/org/qortal/transform/block/BlockTransformer.java +++ b/src/main/java/org/qortal/transform/block/BlockTransformer.java @@ -235,7 +235,7 @@ public class BlockTransformer extends Transformer { // Online accounts timestamp is only present if there are also signatures onlineAccountsTimestamp = byteBuffer.getLong(); - final int signaturesByteLength = onlineAccountsSignaturesCount * Transformer.SIGNATURE_LENGTH; + final int signaturesByteLength = getOnlineAccountSignaturesLength(onlineAccountsSignaturesCount, onlineAccountsCount, timestamp); if (signaturesByteLength > BlockChain.getInstance().getMaxBlockSize()) throw new TransformationException("Byte data too long for online accounts signatures"); @@ -371,7 +371,7 @@ public class BlockTransformer extends Transformer { if (onlineAccountsSignatures != null && onlineAccountsSignatures.length > 0) { // Note: we write the number of signatures, not the number of bytes - bytes.write(Ints.toByteArray(onlineAccountsSignatures.length / Transformer.SIGNATURE_LENGTH)); + bytes.write(Ints.toByteArray(blockData.getOnlineAccountsSignaturesCount())); // We only write online accounts timestamp if we have signatures bytes.write(Longs.toByteArray(blockData.getOnlineAccountsTimestamp())); @@ -511,6 +511,16 @@ public class BlockTransformer extends Transformer { return nonces; } + public static int getOnlineAccountSignaturesLength(int onlineAccountsSignaturesCount, int onlineAccountCount, long blockTimestamp) { + if (blockTimestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + // Once mempow is active, we expect the online account signatures to be appended with the nonce values + return (onlineAccountsSignaturesCount * Transformer.SIGNATURE_LENGTH) + (onlineAccountCount * INT_LENGTH); + } + else { + // Before mempow, only the online account signatures were included (which will likely be a single signature) + return onlineAccountsSignaturesCount * Transformer.SIGNATURE_LENGTH; + } + } public static byte[] extract(byte[] input, int pos, int length) { byte[] output = new byte[length]; From a10e669554f7c353687f2af738b6eab96f63fefc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 11 Sep 2022 11:36:19 +0100 Subject: [PATCH 31/83] Allow nonce to be computed for "next" timestamp if mempow is enabled in settings. --- src/main/java/org/qortal/controller/OnlineAccountsManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 013585ee..254d6168 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -464,7 +464,7 @@ public class OnlineAccountsManager { // 'next' timestamp (prioritize this as it's the most important, if mempow active) final long nextOnlineAccountsTimestamp = toOnlineAccountTimestamp(now) + getOnlineTimestampModulus(); - if (nextOnlineAccountsTimestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + if (isMemoryPoWActive(now)) { boolean success = computeOurAccountsForTimestamp(nextOnlineAccountsTimestamp); if (!success) { // We didn't compute the required nonce value(s), and so can't proceed until they have been retried From f042b5ca5f4c28b858de897be92c7ec1d3d275c5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 11 Sep 2022 11:41:11 +0100 Subject: [PATCH 32/83] If mempow is active, remove any legacy accounts from a to-be-minted block that are missing a nonce. --- src/main/java/org/qortal/block/Block.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 293cb3eb..c21b97d7 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -373,6 +373,11 @@ public class Block { return null; } + // If mempow is active, remove any legacy accounts that are missing a nonce + if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + onlineAccounts.removeIf(a -> a.getNonce() < 0); + } + // Load sorted list of reward share public keys into memory, so that the indexes can be obtained. // This is up to 100x faster than querying each index separately. For 4150 reward share keys, it // was taking around 5000ms to query individually, vs 50ms using this approach. From 063ef8507bc57189942f9ea6979ce45d914fa5f6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 11 Sep 2022 20:04:51 +0100 Subject: [PATCH 33/83] Fix for NPE in last commit --- src/main/java/org/qortal/block/Block.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index c21b97d7..e0581e7d 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -375,7 +375,7 @@ public class Block { // If mempow is active, remove any legacy accounts that are missing a nonce if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { - onlineAccounts.removeIf(a -> a.getNonce() < 0); + onlineAccounts.removeIf(a -> a.getNonce() == null || a.getNonce() < 0); } // Load sorted list of reward share public keys into memory, so that the indexes can be obtained. From 2d29fdca00ccfa34fa3188b192d7a7d0a9403a4e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 16 Sep 2022 11:19:10 +0100 Subject: [PATCH 34/83] Allow BTC trades in redeemAll / refundAll, since most will now be using ACCTv3. --- .../org/qortal/api/resource/CrossChainHtlcResource.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java index cf098f53..9f10f781 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java @@ -373,10 +373,6 @@ public class CrossChainHtlcResource { // Use secret-A to redeem P2SH-A Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain(); - if (bitcoiny.getClass() == Bitcoin.class) { - LOGGER.info("Redeeming a Bitcoin HTLC is not yet supported"); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } int lockTime = crossChainTradeData.lockTimeA; byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTime, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); @@ -599,11 +595,6 @@ public class CrossChainHtlcResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain(); - if (bitcoiny.getClass() == Bitcoin.class) { - LOGGER.info("Refunding a Bitcoin HTLC is not yet supported"); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - int lockTime = tradeBotData.getLockTimeA(); // We can't refund P2SH-A until lockTime-A has passed From aff49e6bdf75d64b03bcc2200c921744c9c6eb6a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Sep 2022 10:30:10 +0100 Subject: [PATCH 35/83] Added support for ARRR refunds via /crosschain/htlc/refund/{ataddress} and /crosschain/htlc/refundAll This could probably be refactored into multiple classes to make the code cleaner, but it is functional for now. --- .../api/resource/CrossChainHtlcResource.java | 74 ++++++++++++++----- .../tradebot/PirateChainACCTv3TradeBot.java | 8 +- .../org/qortal/crosschain/PirateChain.java | 12 +++ .../qortal/crosschain/PirateChainHTLC.java | 10 +-- .../test/crosschain/PirateChainTests.java | 8 +- 5 files changed, 75 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java index 9f10f781..664b013a 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java @@ -1,5 +1,6 @@ package org.qortal.api.resource; +import com.google.common.hash.HashCode; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; @@ -538,11 +539,6 @@ public class CrossChainHtlcResource { try { // Determine foreign blockchain receive address for refund Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain(); - if (Objects.equals(bitcoiny.getCurrencyCode(), "ARRR")) { - LOGGER.info("Skipping AT {} because ARRR is currently unsupported", atAddress); - continue; - } - String receivingAddress = bitcoiny.getUnusedReceiveAddress(tradeBotData.getForeignKey()); LOGGER.info("Attempting to refund P2SH balance associated with AT {}...", atAddress); @@ -585,7 +581,7 @@ public class CrossChainHtlcResource { // If the AT is "finished" then it will have a zero balance // In these cases we should avoid HTLC refunds if tbe QORT haven't been returned to the seller if (atData.getIsFinished() && crossChainTradeData.mode != AcctMode.REFUNDED && crossChainTradeData.mode != AcctMode.CANCELLED) { - LOGGER.info(String.format("Skipping AT %s because the QORT has already been redemed", atAddress)); + LOGGER.info(String.format("Skipping AT %s because the QORT has already been redeemed by the buyer", atAddress)); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } @@ -606,15 +602,26 @@ public class CrossChainHtlcResource { if (medianBlockTime <= lockTime) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_TOO_SOON); - byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTime, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); - String p2shAddressA = bitcoiny.deriveP2shAddress(redeemScriptA); - LOGGER.info(String.format("Refunding P2SH address: %s", p2shAddressA)); - // Fee for redeem/refund is subtracted from P2SH-A balance. long feeTimestamp = calcFeeTimestamp(lockTime, crossChainTradeData.tradeTimeout); long p2shFee = bitcoiny.getP2shFee(feeTimestamp); long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoiny.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + // Create redeem script based on destination chain + byte[] redeemScriptA; + String p2shAddressA; + BitcoinyHTLC.Status htlcStatusA; + if (Objects.equals(bitcoiny.getCurrencyCode(), "ARRR")) { + redeemScriptA = PirateChainHTLC.buildScript(tradeBotData.getTradeForeignPublicKey(), lockTime, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + p2shAddressA = PirateChain.getInstance().deriveP2shAddressBPrefix(redeemScriptA); + htlcStatusA = PirateChainHTLC.determineHtlcStatus(bitcoiny.getBlockchainProvider(), p2shAddressA, minimumAmountA); + } + else { + redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTime, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + p2shAddressA = bitcoiny.deriveP2shAddress(redeemScriptA); + htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoiny.getBlockchainProvider(), p2shAddressA, minimumAmountA); + } + LOGGER.info(String.format("Refunding P2SH address: %s", p2shAddressA)); switch (htlcStatusA) { case UNFUNDED: @@ -631,18 +638,45 @@ public class CrossChainHtlcResource { case FUNDED:{ Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); - ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); - List fundingOutputs = bitcoiny.getUnspentOutputs(p2shAddressA); - // Validate the destination foreign blockchain address - Address receiving = Address.fromString(bitcoiny.getNetworkParameters(), receiveAddress); - if (receiving.getOutputScriptType() != Script.ScriptType.P2PKH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + if (Objects.equals(bitcoiny.getCurrencyCode(), "ARRR")) { + // Pirate Chain custom integration - Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoiny.getNetworkParameters(), refundAmount, refundKey, - fundingOutputs, redeemScriptA, lockTime, receiving.getHash()); + PirateChain pirateChain = PirateChain.getInstance(); + String p2shAddressT3 = pirateChain.deriveP2shAddress(redeemScriptA); + + // Get funding txid + String fundingTxidHex = PirateChainHTLC.getUnspentFundingTxid(pirateChain.getBlockchainProvider(), p2shAddressA, minimumAmountA); + if (fundingTxidHex == null) { + throw new ForeignBlockchainException("Missing funding txid when refunding P2SH"); + } + String fundingTxid58 = Base58.encode(HashCode.fromString(fundingTxidHex).asBytes()); + + byte[] privateKey = tradeBotData.getTradePrivateKey(); + String privateKey58 = Base58.encode(privateKey); + String redeemScript58 = Base58.encode(redeemScriptA); + + String txid = PirateChain.getInstance().refundP2sh(p2shAddressT3, + receiveAddress, refundAmount.value, redeemScript58, fundingTxid58, lockTime, privateKey58); + LOGGER.info("Refund txid: {}", txid); + } + else { + // ElectrumX coins + + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = bitcoiny.getUnspentOutputs(p2shAddressA); + + // Validate the destination foreign blockchain address + Address receiving = Address.fromString(bitcoiny.getNetworkParameters(), receiveAddress); + if (receiving.getOutputScriptType() != Script.ScriptType.P2PKH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoiny.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptA, lockTime, receiving.getHash()); + + bitcoiny.broadcastTransaction(p2shRefundTransaction); + } - bitcoiny.broadcastTransaction(p2shRefundTransaction); return true; } } diff --git a/src/main/java/org/qortal/controller/tradebot/PirateChainACCTv3TradeBot.java b/src/main/java/org/qortal/controller/tradebot/PirateChainACCTv3TradeBot.java index 8f413093..9834df20 100644 --- a/src/main/java/org/qortal/controller/tradebot/PirateChainACCTv3TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/PirateChainACCTv3TradeBot.java @@ -523,7 +523,7 @@ public class PirateChainACCTv3TradeBot implements AcctTradeBot { long p2shFee = PirateChain.getInstance().getP2shFee(feeTimestamp); final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee; - PirateChainHTLC.Status htlcStatusA = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmountA); + BitcoinyHTLC.Status htlcStatusA = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmountA); switch (htlcStatusA) { case UNFUNDED: @@ -613,7 +613,7 @@ public class PirateChainACCTv3TradeBot implements AcctTradeBot { long p2shFee = PirateChain.getInstance().getP2shFee(feeTimestamp); long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - PirateChainHTLC.Status htlcStatusA = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmountA); + BitcoinyHTLC.Status htlcStatusA = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmountA); switch (htlcStatusA) { case UNFUNDED: @@ -751,7 +751,7 @@ public class PirateChainACCTv3TradeBot implements AcctTradeBot { long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; String receivingAddress = Bech32.encode("zs", receivingAccountInfo); - PirateChainHTLC.Status htlcStatusA = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmountA); + BitcoinyHTLC.Status htlcStatusA = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmountA); switch (htlcStatusA) { case UNFUNDED: @@ -822,7 +822,7 @@ public class PirateChainACCTv3TradeBot implements AcctTradeBot { long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); long p2shFee = PirateChain.getInstance().getP2shFee(feeTimestamp); long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - PirateChainHTLC.Status htlcStatusA = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmountA); + BitcoinyHTLC.Status htlcStatusA = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmountA); switch (htlcStatusA) { case UNFUNDED: diff --git a/src/main/java/org/qortal/crosschain/PirateChain.java b/src/main/java/org/qortal/crosschain/PirateChain.java index 97aa07fe..09b37481 100644 --- a/src/main/java/org/qortal/crosschain/PirateChain.java +++ b/src/main/java/org/qortal/crosschain/PirateChain.java @@ -4,6 +4,12 @@ import cash.z.wallet.sdk.rpc.CompactFormats; import com.google.common.hash.HashCode; import com.rust.litewalletjni.LiteWalletJni; import org.bitcoinj.core.*; +import org.bitcoinj.crypto.ChildNumber; +import org.bitcoinj.crypto.DeterministicKey; +import org.bitcoinj.script.Script; +import org.bitcoinj.script.ScriptBuilder; +import org.bitcoinj.wallet.DeterministicKeyChain; +import org.bitcoinj.wallet.Wallet; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -352,6 +358,12 @@ public class PirateChain extends Bitcoiny { } } + public String getUnusedReceiveAddress(String key58) throws ForeignBlockchainException { + // For now, return the main wallet address + // FUTURE: generate an unused one + return this.getWalletAddress(key58); + } + public String sendCoins(PirateChainSendRequest pirateChainSendRequest) throws ForeignBlockchainException { PirateChainWalletController walletController = PirateChainWalletController.getInstance(); walletController.initWithEntropy58(pirateChainSendRequest.entropy58); diff --git a/src/main/java/org/qortal/crosschain/PirateChainHTLC.java b/src/main/java/org/qortal/crosschain/PirateChainHTLC.java index f28897dc..17f7ad74 100644 --- a/src/main/java/org/qortal/crosschain/PirateChainHTLC.java +++ b/src/main/java/org/qortal/crosschain/PirateChainHTLC.java @@ -3,25 +3,17 @@ package org.qortal.crosschain; import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.bitcoinj.core.*; -import org.bitcoinj.core.Transaction.SigHash; -import org.bitcoinj.crypto.TransactionSignature; import org.bitcoinj.script.Script; -import org.bitcoinj.script.ScriptBuilder; import org.bitcoinj.script.ScriptChunk; -import org.bitcoinj.script.ScriptOpCodes; import org.qortal.crypto.Crypto; import org.qortal.utils.Base58; import org.qortal.utils.BitTwiddling; import java.util.*; -import java.util.function.Function; +import static org.qortal.crosschain.BitcoinyHTLC.Status; public class PirateChainHTLC { - public enum Status { - UNFUNDED, FUNDING_IN_PROGRESS, FUNDED, REDEEM_IN_PROGRESS, REDEEMED, REFUND_IN_PROGRESS, REFUNDED - } - public static final int SECRET_LENGTH = 32; public static final int MIN_LOCKTIME = 1500000000; diff --git a/src/test/java/org/qortal/test/crosschain/PirateChainTests.java b/src/test/java/org/qortal/test/crosschain/PirateChainTests.java index d203cf5e..9502e45a 100644 --- a/src/test/java/org/qortal/test/crosschain/PirateChainTests.java +++ b/src/test/java/org/qortal/test/crosschain/PirateChainTests.java @@ -20,7 +20,7 @@ import java.util.Arrays; import java.util.List; import static org.junit.Assert.*; -import static org.qortal.crosschain.PirateChainHTLC.Status.*; +import static org.qortal.crosschain.BitcoinyHTLC.Status.*; public class PirateChainTests extends Common { @@ -121,7 +121,7 @@ public class PirateChainTests extends Common { String p2shAddress = "ba6Q5HWrWtmfU2WZqQbrFdRYsafA45cUAt"; long p2shFee = 10000; final long minimumAmount = 10000 + p2shFee; - PirateChainHTLC.Status htlcStatus = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmount); + BitcoinyHTLC.Status htlcStatus = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmount); assertEquals(FUNDED, htlcStatus); } @@ -130,7 +130,7 @@ public class PirateChainTests extends Common { String p2shAddress = "bYZrzSSgGp8aEGvihukoMGU8sXYrx19Wka"; long p2shFee = 10000; final long minimumAmount = 10000 + p2shFee; - PirateChainHTLC.Status htlcStatus = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmount); + BitcoinyHTLC.Status htlcStatus = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmount); assertEquals(REDEEMED, htlcStatus); } @@ -139,7 +139,7 @@ public class PirateChainTests extends Common { String p2shAddress = "bE49izfVxz8odhu8c2BcUaVFUnt7NLFRgv"; long p2shFee = 10000; final long minimumAmount = 10000 + p2shFee; - PirateChainHTLC.Status htlcStatus = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmount); + BitcoinyHTLC.Status htlcStatus = PirateChainHTLC.determineHtlcStatus(pirateChain.getBlockchainProvider(), p2shAddress, minimumAmount); assertEquals(REFUNDED, htlcStatus); } From 791a9b78ec7abd3765202e46c2a694b5bdf9e123 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Sep 2022 10:36:25 +0100 Subject: [PATCH 36/83] Added support for Pirate Chain wallets on FreeBSD. --- .../org/qortal/controller/PirateChainWalletController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/PirateChainWalletController.java b/src/main/java/org/qortal/controller/PirateChainWalletController.java index 931850db..1eac4b3a 100644 --- a/src/main/java/org/qortal/controller/PirateChainWalletController.java +++ b/src/main/java/org/qortal/controller/PirateChainWalletController.java @@ -238,10 +238,10 @@ public class PirateChainWalletController extends Thread { if (osName.equals("Mac OS X") && osArchitecture.equals("x86_64")) { return "librust-macos-x86_64.dylib"; } - else if (osName.equals("Linux") && osArchitecture.equals("aarch64")) { + else if ((osName.equals("Linux") || osName.equals("FreeBSD")) && osArchitecture.equals("aarch64")) { return "librust-linux-aarch64.so"; } - else if (osName.equals("Linux") && osArchitecture.equals("amd64")) { + else if ((osName.equals("Linux") || osName.equals("FreeBSD")) && osArchitecture.equals("amd64")) { return "librust-linux-x86_64.so"; } else if (osName.contains("Windows") && osArchitecture.equals("amd64")) { From 858269f6cb79275a2a19a7993a0480b3d64f5481 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Sep 2022 12:21:56 +0100 Subject: [PATCH 37/83] ChatTransaction MAX_DATA_SIZE increased from 256 to 1024 bytes, to allow for new UI features. --- src/main/java/org/qortal/transaction/ChatTransaction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/transaction/ChatTransaction.java b/src/main/java/org/qortal/transaction/ChatTransaction.java index 2671c209..9cccd42a 100644 --- a/src/main/java/org/qortal/transaction/ChatTransaction.java +++ b/src/main/java/org/qortal/transaction/ChatTransaction.java @@ -26,7 +26,7 @@ public class ChatTransaction extends Transaction { private ChatTransactionData chatTransactionData; // Other useful constants - public static final int MAX_DATA_SIZE = 256; + public static final int MAX_DATA_SIZE = 1024; public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes public static final int POW_DIFFICULTY_WITH_QORT = 8; // leading zero bits public static final int POW_DIFFICULTY_NO_QORT = 12; // leading zero bits From 02ac6dd8c1dd8a3429d6a9fd715a63fd0818444f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Sep 2022 13:28:32 +0100 Subject: [PATCH 38/83] Added GET /chat/message/{signature} endpoint. This will ease the transition to a Q-Chat protocol, where chat messages will no longer be regular transactions. --- .../org/qortal/api/resource/ChatResource.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/ChatResource.java b/src/main/java/org/qortal/api/resource/ChatResource.java index be8bd7d7..79e479b1 100644 --- a/src/main/java/org/qortal/api/resource/ChatResource.java +++ b/src/main/java/org/qortal/api/resource/ChatResource.java @@ -99,6 +99,38 @@ public class ChatResource { } } + @GET + @Path("/message/{signature}") + @Operation( + summary = "Find chat message by signature", + responses = { + @ApiResponse( + description = "CHAT message", + content = @Content( + schema = @Schema( + implementation = ChatMessage.class + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE}) + public ChatMessage getMessageBySignature(@QueryParam("signature") String signature58) { + byte[] signature = Base58.decode(signature58); + + try (final Repository repository = RepositoryManager.getRepository()) { + + ChatTransactionData chatTransactionData = (ChatTransactionData) repository.getTransactionRepository().fromSignature(signature); + if (chatTransactionData == null) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Message not found"); + } + + return repository.getChatRepository().toChatMessage(chatTransactionData); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + @GET @Path("/active/{address}") @Operation( From 5017072f6c00c91c50825ae4dc1e8a0c48f1d308 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Sep 2022 13:50:04 +0100 Subject: [PATCH 39/83] Use path parameter instead of query string. --- src/main/java/org/qortal/api/resource/ChatResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/ChatResource.java b/src/main/java/org/qortal/api/resource/ChatResource.java index 79e479b1..0bbd1951 100644 --- a/src/main/java/org/qortal/api/resource/ChatResource.java +++ b/src/main/java/org/qortal/api/resource/ChatResource.java @@ -115,7 +115,7 @@ public class ChatResource { } ) @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE}) - public ChatMessage getMessageBySignature(@QueryParam("signature") String signature58) { + public ChatMessage getMessageBySignature(@PathParam("signature") String signature58) { byte[] signature = Base58.decode(signature58); try (final Repository repository = RepositoryManager.getRepository()) { From 64ef8ab8633a584c4a57108274091961a391a7fe Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 19 Sep 2022 16:36:39 +0100 Subject: [PATCH 40/83] OnlineAccountsV3Message.MIN_PEER_VERSION set to 3.6.0 --- .../org/qortal/network/message/OnlineAccountsV3Message.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java index 0c5f6730..d554d96c 100644 --- a/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java +++ b/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java @@ -20,7 +20,7 @@ import java.util.Map; */ public class OnlineAccountsV3Message extends Message { - public static final long MIN_PEER_VERSION = 0x300050001L; // 3.5.1 + public static final long MIN_PEER_VERSION = 0x300060000L; // 3.6.0 private List onlineAccounts; From 952c51ab2526e21ef15bfd97c9029b94afe76408 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 19 Sep 2022 17:27:07 +0100 Subject: [PATCH 41/83] QORA / block reward adjustments set to activate at height 1010000 --- src/main/resources/blockchain.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 8d1600ed..fad81ab5 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -56,7 +56,7 @@ ], "qoraHoldersShareByHeight": [ { "height": 1, "share": 0.20 }, - { "height": 9999999, "share": 0.01 } + { "height": 1010000, "share": 0.01 } ], "qoraPerQortReward": 250, "minAccountsToActivateShareBin": 30, @@ -75,7 +75,7 @@ "atFindNextTransactionFix": 275000, "newBlockSigHeight": 320000, "shareBinFix": 399000, - "sharesByLevelV2Height": 9999999, + "sharesByLevelV2Height": 1010000, "rewardShareLimitTimestamp": 1657382400000, "calcChainWeightTimestamp": 1620579600000, "transactionV5Timestamp": 1642176000000, From b99b1f5d5766b25f995590d40643980b2006e05e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 19 Sep 2022 17:29:26 +0100 Subject: [PATCH 42/83] Bump version to 3.6.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 22017136..e045e0f4 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.5.0 + 3.6.0 jar true From 84d42b93e15ae478672e7fe81602f7ba542bd08e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 20 Sep 2022 08:50:37 +0100 Subject: [PATCH 43/83] Reordered code in Block.mint() to fix potential issue after mempow activates. --- src/main/java/org/qortal/block/Block.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index e0581e7d..bdae83c2 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -368,16 +368,17 @@ public class Block { // Fetch our list of online accounts List onlineAccounts = OnlineAccountsManager.getInstance().getOnlineAccounts(onlineAccountsTimestamp); - if (onlineAccounts.isEmpty()) { - LOGGER.error("No online accounts - not even our own?"); - return null; - } // If mempow is active, remove any legacy accounts that are missing a nonce if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { onlineAccounts.removeIf(a -> a.getNonce() == null || a.getNonce() < 0); } + if (onlineAccounts.isEmpty()) { + LOGGER.error("No online accounts - not even our own?"); + return null; + } + // Load sorted list of reward share public keys into memory, so that the indexes can be obtained. // This is up to 100x faster than querying each index separately. For 4150 reward share keys, it // was taking around 5000ms to query individually, vs 50ms using this approach. From 951c85faf1b743cf5de9da21bb819c71b17449b4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 20 Sep 2022 22:26:30 +0100 Subject: [PATCH 44/83] Fixed bug causing error 500 in some cases. --- .../org/qortal/data/arbitrary/ArbitraryResourceMetadata.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java index 75b5a4d8..e2bcaf56 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java @@ -24,7 +24,10 @@ public class ArbitraryResourceMetadata { this.description = description; this.tags = tags; this.category = category; - this.categoryName = category.getName(); + + if (category != null) { + this.categoryName = category.getName(); + } } public static ArbitraryResourceMetadata fromTransactionMetadata(ArbitraryDataTransactionMetadata transactionMetadata) { From 49d83650f4c202a64b1883a0240cba6884002a00 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 23 Sep 2022 15:25:44 +0100 Subject: [PATCH 45/83] Removed online accounts V2 and V1 messaging, as the V3 format will soon be required due to the nonce values. --- .../org/qortal/controller/Controller.java | 11 +- .../controller/OnlineAccountsManager.java | 157 +----------------- 2 files changed, 6 insertions(+), 162 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 4ff08e15..f6711991 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1232,19 +1232,10 @@ public class Controller extends Thread { break; case GET_ONLINE_ACCOUNTS: - OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsMessage(peer, message); - break; - case ONLINE_ACCOUNTS: - OnlineAccountsManager.getInstance().onNetworkOnlineAccountsMessage(peer, message); - break; - case GET_ONLINE_ACCOUNTS_V2: - OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsV2Message(peer, message); - break; - case ONLINE_ACCOUNTS_V2: - OnlineAccountsManager.getInstance().onNetworkOnlineAccountsV2Message(peer, message); + // No longer supported - to be eventually removed break; case GET_ONLINE_ACCOUNTS_V3: diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 254d6168..b4bfab12 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -55,12 +55,8 @@ public class OnlineAccountsManager { private static final long ONLINE_ACCOUNTS_QUEUE_INTERVAL = 100L; //ms private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms - private static final long ONLINE_ACCOUNTS_LEGACY_BROADCAST_INTERVAL = 60 * 1000L; // ms private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 5 * 1000L; // ms - private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L; // v3.2.0 - private static final long ONLINE_ACCOUNTS_V3_PEER_VERSION = 0x0300040000L; // v3.4.0 - // MemoryPoW public final int POW_BUFFER_SIZE = 1 * 1024 * 1024; // bytes public int POW_DIFFICULTY = 18; // leading zero bits @@ -125,9 +121,7 @@ public class OnlineAccountsManager { // Send our online accounts executor.scheduleAtFixedRate(this::sendOurOnlineAccountsInfo, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS); - // Request online accounts from peers (legacy) - executor.scheduleAtFixedRate(this::requestLegacyRemoteOnlineAccounts, ONLINE_ACCOUNTS_LEGACY_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_LEGACY_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS); - // Request online accounts from peers (V3+) + // Request online accounts from peers executor.scheduleAtFixedRate(this::requestRemoteOnlineAccounts, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS); // Process import queue @@ -399,30 +393,7 @@ public class OnlineAccountsManager { } /** - * Request data from other peers. (Pre-V3) - */ - private void requestLegacyRemoteOnlineAccounts() { - final Long now = NTP.getTime(); - if (now == null) - return; - - // Don't bother if we're not up to date - if (!Controller.getInstance().isUpToDate()) - return; - - List mergedOnlineAccounts = Set.copyOf(this.currentOnlineAccounts.values()).stream().flatMap(Set::stream).collect(Collectors.toList()); - - Message messageV2 = new GetOnlineAccountsV2Message(mergedOnlineAccounts); - - Network.getInstance().broadcast(peer -> - peer.getPeersVersion() < ONLINE_ACCOUNTS_V3_PEER_VERSION - ? messageV2 - : null - ); - } - - /** - * Request data from other peers. V3+ + * Request data from other peers */ private void requestRemoteOnlineAccounts() { final Long now = NTP.getTime(); @@ -435,11 +406,7 @@ public class OnlineAccountsManager { Message messageV3 = new GetOnlineAccountsV3Message(currentOnlineAccountsHashes); - Network.getInstance().broadcast(peer -> - peer.getPeersVersion() >= ONLINE_ACCOUNTS_V3_PEER_VERSION - ? messageV3 - : null - ); + Network.getInstance().broadcast(peer -> messageV3); } /** @@ -579,17 +546,7 @@ public class OnlineAccountsManager { if (!hasInfoChanged) return false; - Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts); - Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts); - Message messageV3 = new OnlineAccountsV3Message(ourOnlineAccounts); - - Network.getInstance().broadcast(peer -> - peer.getPeersVersion() >= OnlineAccountsV3Message.MIN_PEER_VERSION - ? messageV3 - : peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION - ? messageV2 - : messageV1 - ); + Network.getInstance().broadcast(peer -> new OnlineAccountsV3Message(ourOnlineAccounts)); LOGGER.debug("Broadcasted {} online account{} with timestamp {}", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp); @@ -767,106 +724,6 @@ public class OnlineAccountsManager { // Network handlers - public void onNetworkGetOnlineAccountsMessage(Peer peer, Message message) { - GetOnlineAccountsMessage getOnlineAccountsMessage = (GetOnlineAccountsMessage) message; - - List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); - - // Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts - List accountsToSend = Set.copyOf(this.currentOnlineAccounts.values()).stream().flatMap(Set::stream).collect(Collectors.toList()); - int prefilterSize = accountsToSend.size(); - - Iterator iterator = accountsToSend.iterator(); - while (iterator.hasNext()) { - OnlineAccountData onlineAccountData = iterator.next(); - - for (OnlineAccountData excludeAccountData : excludeAccounts) { - if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) { - iterator.remove(); - break; - } - } - } - - if (accountsToSend.isEmpty()) - return; - - Message onlineAccountsMessage = new OnlineAccountsMessage(accountsToSend); - peer.sendMessage(onlineAccountsMessage); - - LOGGER.debug("Sent {} of our {} online accounts to {}", accountsToSend.size(), prefilterSize, peer); - } - - public void onNetworkOnlineAccountsMessage(Peer peer, Message message) { - OnlineAccountsMessage onlineAccountsMessage = (OnlineAccountsMessage) message; - - List peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts(); - LOGGER.debug("Received {} online accounts from {}", peersOnlineAccounts.size(), peer); - - int importCount = 0; - - // Add any online accounts to the queue that aren't already present - for (OnlineAccountData onlineAccountData : peersOnlineAccounts) { - boolean isNewEntry = onlineAccountsImportQueue.add(onlineAccountData); - - if (isNewEntry) - importCount++; - } - - if (importCount > 0) - LOGGER.debug("Added {} online accounts to queue", importCount); - } - - public void onNetworkGetOnlineAccountsV2Message(Peer peer, Message message) { - GetOnlineAccountsV2Message getOnlineAccountsMessage = (GetOnlineAccountsV2Message) message; - - List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); - - // Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts - List accountsToSend = Set.copyOf(this.currentOnlineAccounts.values()).stream().flatMap(Set::stream).collect(Collectors.toList()); - int prefilterSize = accountsToSend.size(); - - Iterator iterator = accountsToSend.iterator(); - while (iterator.hasNext()) { - OnlineAccountData onlineAccountData = iterator.next(); - - for (OnlineAccountData excludeAccountData : excludeAccounts) { - if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) { - iterator.remove(); - break; - } - } - } - - if (accountsToSend.isEmpty()) - return; - - Message onlineAccountsMessage = new OnlineAccountsV2Message(accountsToSend); - peer.sendMessage(onlineAccountsMessage); - - LOGGER.debug("Sent {} of our {} online accounts to {}", accountsToSend.size(), prefilterSize, peer); - } - - public void onNetworkOnlineAccountsV2Message(Peer peer, Message message) { - OnlineAccountsV2Message onlineAccountsMessage = (OnlineAccountsV2Message) message; - - List peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts(); - LOGGER.debug("Received {} online accounts from {}", peersOnlineAccounts.size(), peer); - - int importCount = 0; - - // Add any online accounts to the queue that aren't already present - for (OnlineAccountData onlineAccountData : peersOnlineAccounts) { - boolean isNewEntry = onlineAccountsImportQueue.add(onlineAccountData); - - if (isNewEntry) - importCount++; - } - - if (importCount > 0) - LOGGER.debug("Added {} online accounts to queue", importCount); - } - public void onNetworkGetOnlineAccountsV3Message(Peer peer, Message message) { GetOnlineAccountsV3Message getOnlineAccountsMessage = (GetOnlineAccountsV3Message) message; @@ -920,11 +777,7 @@ public class OnlineAccountsManager { } } - peer.sendMessage( - peer.getPeersVersion() >= OnlineAccountsV3Message.MIN_PEER_VERSION ? - new OnlineAccountsV3Message(outgoingOnlineAccounts) : - new OnlineAccountsV2Message(outgoingOnlineAccounts) - ); + peer.sendMessage(new OnlineAccountsV3Message(outgoingOnlineAccounts)); LOGGER.debug("Sent {} online accounts to {}", outgoingOnlineAccounts.size(), peer); } From 84a16157d1e7407718b4439d9e1e1afdc629f108 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 23 Sep 2022 18:02:46 +0100 Subject: [PATCH 46/83] Don't add online accounts to the import queue if they are already validated --- .../java/org/qortal/controller/OnlineAccountsManager.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index b4bfab12..eaf12db3 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -792,6 +792,12 @@ public class OnlineAccountsManager { // Add any online accounts to the queue that aren't already present for (OnlineAccountData onlineAccountData : peersOnlineAccounts) { + + Set onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountData.getTimestamp(), k -> ConcurrentHashMap.newKeySet()); + if (onlineAccounts.contains(onlineAccountData)) + // We have already validated this online account + continue; + boolean isNewEntry = onlineAccountsImportQueue.add(onlineAccountData); if (isNewEntry) From 99858f378100ee8dea6103f1aed6d15219f730d7 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 23 Sep 2022 18:28:41 +0100 Subject: [PATCH 47/83] Wait 30 seconds after the node starts before computing our online accounts. This allows some time for initial online account lists to be retrieved, and reduces the chances of the same nonce being computed twice. --- .../controller/OnlineAccountsManager.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index eaf12db3..39ce8a85 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -57,6 +57,8 @@ public class OnlineAccountsManager { private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 5 * 1000L; // ms + private static final long INITIAL_SLEEP_INTERVAL = 30 * 1000L; + // MemoryPoW public final int POW_BUFFER_SIZE = 1 * 1024 * 1024; // bytes public int POW_DIFFICULTY = 18; // leading zero bits @@ -118,14 +120,23 @@ public class OnlineAccountsManager { // Expire old online accounts signatures executor.scheduleAtFixedRate(this::expireOldOnlineAccounts, ONLINE_ACCOUNTS_TASKS_INTERVAL, ONLINE_ACCOUNTS_TASKS_INTERVAL, TimeUnit.MILLISECONDS); - // Send our online accounts - executor.scheduleAtFixedRate(this::sendOurOnlineAccountsInfo, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS); - // Request online accounts from peers executor.scheduleAtFixedRate(this::requestRemoteOnlineAccounts, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS); // Process import queue executor.scheduleWithFixedDelay(this::processOnlineAccountsImportQueue, ONLINE_ACCOUNTS_QUEUE_INTERVAL, ONLINE_ACCOUNTS_QUEUE_INTERVAL, TimeUnit.MILLISECONDS); + + // Sleep for some time before scheduling sendOurOnlineAccountsInfo() + // This allows some time for initial online account lists to be retrieved, and + // reduces the chances of the same nonce being computed twice + try { + Thread.sleep(INITIAL_SLEEP_INTERVAL); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + // Send our online accounts + executor.scheduleAtFixedRate(this::sendOurOnlineAccountsInfo, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS); } public void shutdown() { From 6d9e6e8d4c89582ffad23c1094b6a8e3aee91116 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 23 Sep 2022 18:46:01 +0100 Subject: [PATCH 48/83] Allow duplicate variations of each OnlineAccountData in the import queue, but don't allow two entries that match exactly. --- .../controller/OnlineAccountsManager.java | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 39ce8a85..f770bc3a 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -66,7 +66,7 @@ public class OnlineAccountsManager { private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(4, new NamedThreadFactory("OnlineAccounts")); private volatile boolean isStopping = false; - private final Set onlineAccountsImportQueue = ConcurrentHashMap.newKeySet(); + private final List onlineAccountsImportQueue = Collections.synchronizedList(new ArrayList<>()); /** * Cache of 'current' online accounts, keyed by timestamp @@ -184,9 +184,12 @@ public class OnlineAccountsManager { LOGGER.debug("Processing online accounts import queue (size: {})", this.onlineAccountsImportQueue.size()); + // Take a copy of onlineAccountsImportQueue so we can safely remove whilst iterating + List onlineAccountsImportQueueCopy = new ArrayList<>(this.onlineAccountsImportQueue); + Set onlineAccountsToAdd = new HashSet<>(); try (final Repository repository = RepositoryManager.getRepository()) { - for (OnlineAccountData onlineAccountData : this.onlineAccountsImportQueue) { + for (OnlineAccountData onlineAccountData : onlineAccountsImportQueueCopy) { if (isStopping) return; @@ -207,6 +210,19 @@ public class OnlineAccountsManager { } } + private boolean importQueueContainsExactMatch(OnlineAccountData acc) { + // Check if an item exists where all properties match exactly + // This is needed because signature and nonce are not compared in OnlineAccountData.equals() + synchronized (onlineAccountsImportQueue) { + return onlineAccountsImportQueue.stream().anyMatch(otherAcc -> + acc.getTimestamp() == otherAcc.getTimestamp() && + Arrays.equals(acc.getPublicKey(), otherAcc.getPublicKey()) && + acc.getNonce() == otherAcc.getNonce() && + Arrays.equals(acc.getSignature(), otherAcc.getSignature()) + ); + } + } + /** * Check if supplied onlineAccountData is superior (i.e. has a nonce value) than existing record. * Two entries are considered equal even if the nonce differs, to prevent multiple variations @@ -809,6 +825,10 @@ public class OnlineAccountsManager { // We have already validated this online account continue; + if (this.importQueueContainsExactMatch(onlineAccountData)) + // Identical online account data already present in queue + continue; + boolean isNewEntry = onlineAccountsImportQueue.add(onlineAccountData); if (isNewEntry) From ea4f4d949bdefac88736207b84b16a4d2229412d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 23 Sep 2022 19:45:59 +0100 Subject: [PATCH 49/83] When validating online accounts, enforce mempow if the online account's timestamp is after the feature trigger. --- .../java/org/qortal/controller/OnlineAccountsManager.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index f770bc3a..4d1ab561 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -325,8 +325,9 @@ public class OnlineAccountsManager { return false; } - // Validate mempow if feature trigger is active - if (now >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { + // Validate mempow if feature trigger is active (or if online account's timestamp is past the trigger timestamp) + long memoryPoWStartTimestamp = BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp(); + if (now >= memoryPoWStartTimestamp || onlineAccountTimestamp >= memoryPoWStartTimestamp) { if (!getInstance().verifyMemoryPoW(onlineAccountData, now)) { LOGGER.trace(() -> String.format("Rejecting online reward-share for account %s due to invalid PoW nonce", mintingAccount.getAddress())); return false; @@ -628,7 +629,8 @@ public class OnlineAccountsManager { } public boolean verifyMemoryPoW(OnlineAccountData onlineAccountData, Long timestamp) { - if (!isMemoryPoWActive(timestamp)) { + long memoryPoWStartTimestamp = BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp(); + if (timestamp < memoryPoWStartTimestamp && onlineAccountData.getTimestamp() < memoryPoWStartTimestamp) { // Not active yet, so treat it as valid return true; } From c7cf33ef7838bf96777af0e3fab77e67c4e6cf1c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Sep 2022 10:23:55 +0100 Subject: [PATCH 50/83] Set hasOurOnlineAccounts to true if one of our accounts is found before signing. --- src/main/java/org/qortal/controller/OnlineAccountsManager.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 4d1ab561..32d0a47a 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -517,6 +517,8 @@ public class OnlineAccountsManager { Set onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountsTimestamp, k -> ConcurrentHashMap.newKeySet()); boolean alreadyExists = onlineAccounts.stream().anyMatch(a -> Arrays.equals(a.getPublicKey(), publicKey)); if (alreadyExists) { + this.hasOurOnlineAccounts = true; + if (remaining > 0) { // Move on to next account continue; From 174a779e4cd9bc692ef1ef8ecc03edb20206cc04 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Sep 2022 10:56:52 +0100 Subject: [PATCH 51/83] Add accounts from the import queue individually, and then skip future duplicates before unnecessarily validating them again. This closes a gap where accounts would be moved from onlineAccountsImportQueue to onlineAccountsToAdd, but not yet imported. During this time, there was nothing to stop them from being added to the import queue again, causing duplicate validations. --- .../controller/OnlineAccountsManager.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 32d0a47a..de8cfb12 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -193,9 +193,17 @@ public class OnlineAccountsManager { if (isStopping) return; + // Skip this account if it's already validated + Set onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountData.getTimestamp(), k -> ConcurrentHashMap.newKeySet()); + if (onlineAccounts.contains(onlineAccountData)) { + // We have already validated this online account + onlineAccountsImportQueue.remove(onlineAccountData); + continue; + } + boolean isValid = this.isValidCurrentAccount(repository, onlineAccountData); if (isValid) - onlineAccountsToAdd.add(onlineAccountData); + addAccounts(Arrays.asList(onlineAccountData)); // Remove from queue onlineAccountsImportQueue.remove(onlineAccountData); @@ -203,11 +211,6 @@ public class OnlineAccountsManager { } catch (DataException e) { LOGGER.error("Repository issue while verifying online accounts", e); } - - if (!onlineAccountsToAdd.isEmpty()) { - LOGGER.debug("Merging {} validated online accounts from import queue", onlineAccountsToAdd.size()); - addAccounts(onlineAccountsToAdd); - } } private boolean importQueueContainsExactMatch(OnlineAccountData acc) { @@ -381,7 +384,7 @@ public class OnlineAccountsManager { } } - LOGGER.debug(String.format("we have online accounts for timestamps: %s", String.join(", ", this.currentOnlineAccounts.keySet().stream().map(l -> Long.toString(l)).collect(Collectors.joining(", "))))); + LOGGER.trace(String.format("we have online accounts for timestamps: %s", String.join(", ", this.currentOnlineAccounts.keySet().stream().map(l -> Long.toString(l)).collect(Collectors.joining(", "))))); return true; } From 5b81b30974b4fba01bbf796e45d2bb4bbdd7b185 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Sep 2022 13:02:27 +0100 Subject: [PATCH 52/83] Modified online accounts request interval, and introduced bursting. It will now request online accounts every 1 minute instead of every 5 seconds, except for the first 5 minutes following a new online accounts timestamp, in which it will request every 5 seconds (referred to as the "burst" interval). It will also use the burst interval for the first 5 minutes after the node starts. This is based on the idea that most online accounts arrive soon after a new timestamp begins, and so there is no need to request accounts so frequently after that. This should reduce data usage by a significant amount. Once mempow is fully rolled out, the "burst" feature can be reduced or removed, since online accounts will be sent ahead of time, generally 15-30 mins prior to the new online accounts timestamp becoming active. --- .../org/qortal/controller/Controller.java | 4 +++ .../controller/OnlineAccountsManager.java | 31 ++++++++++++++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index f6711991..8e1dfd8a 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -317,6 +317,10 @@ public class Controller extends Thread { } } + public static long uptime() { + return System.currentTimeMillis() - Controller.startTime; + } + /** Returns highest block, or null if it's not available. */ public BlockData getChainTip() { synchronized (this.latestBlocks) { diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index de8cfb12..a0f4db68 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -55,7 +55,12 @@ public class OnlineAccountsManager { private static final long ONLINE_ACCOUNTS_QUEUE_INTERVAL = 100L; //ms private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms - private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 5 * 1000L; // ms + private static final long ONLINE_ACCOUNTS_COMPUTE_INTERVAL = 5 * 1000L; // ms + private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 60 * 1000L; // ms + // After switching to a new online timestamp, we "burst" the online accounts requests + // at an increased interval for a specified amount of time + private static final long ONLINE_ACCOUNTS_BROADCAST_BURST_INTERVAL = 5 * 1000L; // ms + private static final long ONLINE_ACCOUNTS_BROADCAST_BURST_LENGTH = 5 * 60 * 1000L; // ms private static final long INITIAL_SLEEP_INTERVAL = 30 * 1000L; @@ -83,6 +88,8 @@ public class OnlineAccountsManager { */ private final SortedMap> latestBlocksOnlineAccounts = new ConcurrentSkipListMap<>(); + private long lastOnlineAccountsRequest = 0; + private boolean hasOurOnlineAccounts = false; public static long getOnlineTimestampModulus() { @@ -121,7 +128,7 @@ public class OnlineAccountsManager { executor.scheduleAtFixedRate(this::expireOldOnlineAccounts, ONLINE_ACCOUNTS_TASKS_INTERVAL, ONLINE_ACCOUNTS_TASKS_INTERVAL, TimeUnit.MILLISECONDS); // Request online accounts from peers - executor.scheduleAtFixedRate(this::requestRemoteOnlineAccounts, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS); + executor.scheduleAtFixedRate(this::requestRemoteOnlineAccounts, ONLINE_ACCOUNTS_BROADCAST_BURST_INTERVAL, ONLINE_ACCOUNTS_BROADCAST_BURST_INTERVAL, TimeUnit.MILLISECONDS); // Process import queue executor.scheduleWithFixedDelay(this::processOnlineAccountsImportQueue, ONLINE_ACCOUNTS_QUEUE_INTERVAL, ONLINE_ACCOUNTS_QUEUE_INTERVAL, TimeUnit.MILLISECONDS); @@ -136,7 +143,7 @@ public class OnlineAccountsManager { } // Send our online accounts - executor.scheduleAtFixedRate(this::sendOurOnlineAccountsInfo, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS); + executor.scheduleAtFixedRate(this::sendOurOnlineAccountsInfo, ONLINE_ACCOUNTS_COMPUTE_INTERVAL, ONLINE_ACCOUNTS_COMPUTE_INTERVAL, TimeUnit.MILLISECONDS); } public void shutdown() { @@ -435,8 +442,24 @@ public class OnlineAccountsManager { if (!Controller.getInstance().isUpToDate()) return; - Message messageV3 = new GetOnlineAccountsV3Message(currentOnlineAccountsHashes); + long onlineAccountsTimestamp = getCurrentOnlineAccountTimestamp(); + if (now - onlineAccountsTimestamp >= ONLINE_ACCOUNTS_BROADCAST_BURST_LENGTH) { + // New online timestamp started more than 5 mins ago - we probably don't need to request so frequently + if (Controller.uptime() < ONLINE_ACCOUNTS_BROADCAST_BURST_LENGTH) { + // The node recently started up, so we should request at the burst interval + // This could allow accounts to move around the network more easily when an auto update is occurring + } + else if (now - lastOnlineAccountsRequest < ONLINE_ACCOUNTS_BROADCAST_INTERVAL) { + // We already requested online accounts in the last minute, so no need to request again + return; + } + } + + LOGGER.info("Requesting online accounts via broadcast..."); + + lastOnlineAccountsRequest = now; + Message messageV3 = new GetOnlineAccountsV3Message(currentOnlineAccountsHashes); Network.getInstance().broadcast(peer -> messageV3); } From 863a5eff9735671f24f72a03fafc9fb41886bd88 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Sep 2022 13:11:28 +0100 Subject: [PATCH 53/83] Moved various online accounts logs to TRACE level, to make it easier to monitor the queue processing when in DEBUG. --- .../org/qortal/controller/OnlineAccountsManager.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index a0f4db68..40192876 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -365,7 +365,7 @@ public class OnlineAccountsManager { for (var entry : hashesToRebuild.entrySet()) { Long timestamp = entry.getKey(); - LOGGER.debug(() -> String.format("Rehashing for timestamp %d and leading bytes %s", + LOGGER.trace(() -> String.format("Rehashing for timestamp %d and leading bytes %s", timestamp, entry.getValue().stream().sorted(Byte::compareUnsigned).map(leadingByte -> String.format("%02x", leadingByte)).collect(Collectors.joining(", ")) ) @@ -456,7 +456,7 @@ public class OnlineAccountsManager { } } - LOGGER.info("Requesting online accounts via broadcast..."); + LOGGER.debug("Requesting online accounts via broadcast..."); lastOnlineAccountsRequest = now; Message messageV3 = new GetOnlineAccountsV3Message(currentOnlineAccountsHashes); @@ -801,7 +801,7 @@ public class OnlineAccountsManager { Set timestampsOnlineAccounts = this.currentOnlineAccounts.getOrDefault(timestamp, Collections.emptySet()); outgoingOnlineAccounts.addAll(timestampsOnlineAccounts); - LOGGER.debug(() -> String.format("Going to send all %d online accounts for timestamp %d", timestampsOnlineAccounts.size(), timestamp)); + LOGGER.trace(() -> String.format("Going to send all %d online accounts for timestamp %d", timestampsOnlineAccounts.size(), timestamp)); } else { // Quick cache of which leading bytes to send so we only have to filter once Set outgoingLeadingBytes = new HashSet<>(); @@ -825,7 +825,7 @@ public class OnlineAccountsManager { .forEach(outgoingOnlineAccounts::add); if (outgoingOnlineAccounts.size() > beforeAddSize) - LOGGER.debug(String.format("Going to send %d online accounts for timestamp %d and leading bytes %s", + LOGGER.trace(String.format("Going to send %d online accounts for timestamp %d and leading bytes %s", outgoingOnlineAccounts.size() - beforeAddSize, timestamp, outgoingLeadingBytes.stream().sorted(Byte::compareUnsigned).map(leadingByte -> String.format("%02x", leadingByte)).collect(Collectors.joining(", ")) @@ -836,14 +836,14 @@ public class OnlineAccountsManager { peer.sendMessage(new OnlineAccountsV3Message(outgoingOnlineAccounts)); - LOGGER.debug("Sent {} online accounts to {}", outgoingOnlineAccounts.size(), peer); + LOGGER.trace("Sent {} online accounts to {}", outgoingOnlineAccounts.size(), peer); } public void onNetworkOnlineAccountsV3Message(Peer peer, Message message) { OnlineAccountsV3Message onlineAccountsMessage = (OnlineAccountsV3Message) message; List peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts(); - LOGGER.debug("Received {} online accounts from {}", peersOnlineAccounts.size(), peer); + LOGGER.trace("Received {} online accounts from {}", peersOnlineAccounts.size(), peer); int importCount = 0; From 94cdc10151669c92b7daedbce43e47c5212a01d8 Mon Sep 17 00:00:00 2001 From: catbref Date: Tue, 31 May 2022 21:06:34 +0100 Subject: [PATCH 54/83] Initial work on BLOCK_SUMMARIES_V2, part of a bigger arc to improve synchronization. Touches quite a few files because: * Deprecate HEIGHT_V2 because it doesn't contain enough info to be fully useful during sync. Newer peers will re-use BLOCK_SUMMARIES_V2. * For newer peers, instead of sending / broadcasting HEIGHT_V2, send top N block summaries instead, to avoid requests for minor reorgs. * When responding to GET_BLOCK, and we don't actually have the requested block, we currently send an empty BLOCK_SUMMARIES message instead of not responding, which would cause a slow timeout in Synchronizer. This pattern has spread to other network message response code, so now we introduce a generic 'unknown' message type for all these cases. * Remove PeerChainTipData class entirely and re-use BlockSummaryData instead. * Each Peer instance used to hold PeerChainTipData - essentially single latest block summary - but now holds a List of latest block summaries. * PeerChainTipData getter/setter methods modified for compatibility at this point in time. * Repository methods that return BlockSummaryData (or lists of) now try to fully populate them, including newly added block reference field. * Re-worked Peer.canUseCommonBlockData() to be more readable * Cherry-picked patch to Message.fromByteBuffer() to pass an empty, read-only ByteBuffer to subclass fromByteBuffer() methods, instead of null. This allows natural use of BufferUnderflowException if a subclass tries to use read(), or hasRemaining(), etc. from an empty data-payload message. Previously this could have caused an NPE. --- .../org/qortal/api/model/ConnectedPeer.java | 10 +- .../org/qortal/controller/BlockMinter.java | 16 +-- .../org/qortal/controller/Controller.java | 102 +++++++++++------ .../org/qortal/controller/Synchronizer.java | 44 ++++---- .../arbitrary/ArbitraryDataFileManager.java | 7 +- .../qortal/data/block/BlockSummaryData.java | 24 +++- .../qortal/data/block/CommonBlockData.java | 8 +- .../qortal/data/network/PeerChainTipData.java | 37 ------- src/main/java/org/qortal/network/Network.java | 62 +++++++++-- src/main/java/org/qortal/network/Peer.java | 66 ++++++----- .../message/BlockSummariesV2Message.java | 104 ++++++++++++++++++ .../message/GenericUnknownMessage.java | 23 ++++ .../qortal/network/message/MessageType.java | 2 + .../hsqldb/HSQLDBBlockArchiveRepository.java | 8 +- .../hsqldb/HSQLDBBlockRepository.java | 13 ++- 15 files changed, 367 insertions(+), 159 deletions(-) delete mode 100644 src/main/java/org/qortal/data/network/PeerChainTipData.java create mode 100644 src/main/java/org/qortal/network/message/BlockSummariesV2Message.java create mode 100644 src/main/java/org/qortal/network/message/GenericUnknownMessage.java diff --git a/src/main/java/org/qortal/api/model/ConnectedPeer.java b/src/main/java/org/qortal/api/model/ConnectedPeer.java index 21bfc1f9..3d383321 100644 --- a/src/main/java/org/qortal/api/model/ConnectedPeer.java +++ b/src/main/java/org/qortal/api/model/ConnectedPeer.java @@ -1,7 +1,7 @@ package org.qortal.api.model; import io.swagger.v3.oas.annotations.media.Schema; -import org.qortal.data.network.PeerChainTipData; +import org.qortal.data.block.BlockSummaryData; import org.qortal.data.network.PeerData; import org.qortal.network.Handshake; import org.qortal.network.Peer; @@ -63,11 +63,11 @@ public class ConnectedPeer { this.age = "connecting..."; } - PeerChainTipData peerChainTipData = peer.getChainTipData(); + BlockSummaryData peerChainTipData = peer.getChainTipData(); if (peerChainTipData != null) { - this.lastHeight = peerChainTipData.getLastHeight(); - this.lastBlockSignature = peerChainTipData.getLastBlockSignature(); - this.lastBlockTimestamp = peerChainTipData.getLastBlockTimestamp(); + this.lastHeight = peerChainTipData.getHeight(); + this.lastBlockSignature = peerChainTipData.getSignature(); + this.lastBlockTimestamp = peerChainTipData.getTimestamp(); } } diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 343ab4af..a07d37fe 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -26,6 +26,9 @@ import org.qortal.data.block.CommonBlockData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; import org.qortal.network.Peer; +import org.qortal.network.message.BlockSummariesV2Message; +import org.qortal.network.message.HeightV2Message; +import org.qortal.network.message.Message; import org.qortal.repository.BlockRepository; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -431,16 +434,9 @@ public class BlockMinter extends Thread { blockchainLock.unlock(); } - if (newBlockMinted) { - // Broadcast our new chain to network - BlockData newBlockData = newBlock.getBlockData(); - - Network network = Network.getInstance(); - network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newBlockData)); - } - } catch (InterruptedException e) { - // We've been interrupted - time to exit - return; + if (newBlockMinted) { + // Broadcast our new chain to network + Network.getInstance().broadcastOurChain(); } } } catch (DataException e) { diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 8e1dfd8a..ce994757 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -45,7 +45,6 @@ import org.qortal.data.account.AccountData; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.naming.NameData; -import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; import org.qortal.data.transaction.ChatTransactionData; import org.qortal.data.transaction.TransactionData; @@ -731,25 +730,25 @@ public class Controller extends Thread { public static final Predicate hasNoRecentBlock = peer -> { final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp(); - final PeerChainTipData peerChainTipData = peer.getChainTipData(); - return peerChainTipData == null || peerChainTipData.getLastBlockTimestamp() == null || peerChainTipData.getLastBlockTimestamp() < minLatestBlockTimestamp; + final BlockSummaryData peerChainTipData = peer.getChainTipData(); + return peerChainTipData == null || peerChainTipData.getTimestamp() == null || peerChainTipData.getTimestamp() < minLatestBlockTimestamp; }; public static final Predicate hasNoOrSameBlock = peer -> { final BlockData latestBlockData = getInstance().getChainTip(); - final PeerChainTipData peerChainTipData = peer.getChainTipData(); - return peerChainTipData == null || peerChainTipData.getLastBlockSignature() == null || Arrays.equals(latestBlockData.getSignature(), peerChainTipData.getLastBlockSignature()); + final BlockSummaryData peerChainTipData = peer.getChainTipData(); + return peerChainTipData == null || peerChainTipData.getSignature() == null || Arrays.equals(latestBlockData.getSignature(), peerChainTipData.getSignature()); }; public static final Predicate hasOnlyGenesisBlock = peer -> { - final PeerChainTipData peerChainTipData = peer.getChainTipData(); - return peerChainTipData == null || peerChainTipData.getLastHeight() == null || peerChainTipData.getLastHeight() == 1; + final BlockSummaryData peerChainTipData = peer.getChainTipData(); + return peerChainTipData == null || peerChainTipData.getHeight() == 1; }; public static final Predicate hasInferiorChainTip = peer -> { - final PeerChainTipData peerChainTipData = peer.getChainTipData(); + final BlockSummaryData peerChainTipData = peer.getChainTipData(); final List inferiorChainTips = Synchronizer.getInstance().inferiorChainSignatures; - return peerChainTipData == null || peerChainTipData.getLastBlockSignature() == null || inferiorChainTips.contains(ByteArray.wrap(peerChainTipData.getLastBlockSignature())); + return peerChainTipData == null || peerChainTipData.getSignature() == null || inferiorChainTips.contains(ByteArray.wrap(peerChainTipData.getSignature())); }; public static final Predicate hasOldVersion = peer -> { @@ -1011,8 +1010,7 @@ public class Controller extends Thread { network.broadcast(peer -> peer.isOutbound() ? network.buildPeersMessage(peer) : new GetPeersMessage()); // Send our current height - BlockData latestBlockData = getChainTip(); - network.broadcast(peer -> network.buildHeightMessage(peer, latestBlockData)); + network.broadcastOurChain(); // Request unconfirmed transaction signatures, but only if we're up-to-date. // If we're NOT up-to-date then priority is synchronizing first @@ -1219,6 +1217,10 @@ public class Controller extends Thread { onNetworkHeightV2Message(peer, message); break; + case BLOCK_SUMMARIES_V2: + onNetworkBlockSummariesV2Message(peer, message); + break; + case GET_TRANSACTION: TransactionImporter.getInstance().onNetworkGetTransactionMessage(peer, message); break; @@ -1373,8 +1375,10 @@ public class Controller extends Thread { // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout LOGGER.debug(() -> String.format("Sending 'block unknown' response to peer %s for GET_BLOCK request for unknown block %s", peer, Base58.encode(signature))); - // We'll send empty block summaries message as it's very short - Message blockUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + // Send generic 'unknown' message as it's very short + Message blockUnknownMessage = peer.getPeersVersion() >= GenericUnknownMessage.MINIMUM_PEER_VERSION + ? new GenericUnknownMessage() + : new BlockSummariesMessage(Collections.emptyList()); blockUnknownMessage.setId(message.getId()); if (!peer.sendMessage(blockUnknownMessage)) peer.disconnect("failed to send block-unknown response"); @@ -1423,11 +1427,15 @@ public class Controller extends Thread { this.stats.getBlockSummariesStats.requests.incrementAndGet(); // If peer's parent signature matches our latest block signature - // then we can short-circuit with an empty response + // then we have no blocks after that and can short-circuit with an empty response BlockData chainTip = getChainTip(); if (chainTip != null && Arrays.equals(parentSignature, chainTip.getSignature())) { - Message blockSummariesMessage = new BlockSummariesMessage(Collections.emptyList()); + Message blockSummariesMessage = peer.getPeersVersion() >= BlockSummariesV2Message.MINIMUM_PEER_VERSION + ? new BlockSummariesV2Message(Collections.emptyList()) + : new BlockSummariesMessage(Collections.emptyList()); + blockSummariesMessage.setId(message.getId()); + if (!peer.sendMessage(blockSummariesMessage)) peer.disconnect("failed to send block summaries"); @@ -1483,7 +1491,9 @@ public class Controller extends Thread { this.stats.getBlockSummariesStats.fullyFromCache.incrementAndGet(); } - Message blockSummariesMessage = new BlockSummariesMessage(blockSummaries); + Message blockSummariesMessage = peer.getPeersVersion() >= BlockSummariesV2Message.MINIMUM_PEER_VERSION + ? new BlockSummariesV2Message(blockSummaries) + : new BlockSummariesMessage(blockSummaries); blockSummariesMessage.setId(message.getId()); if (!peer.sendMessage(blockSummariesMessage)) peer.disconnect("failed to send block summaries"); @@ -1558,18 +1568,48 @@ public class Controller extends Thread { // If peer is inbound and we've not updated their height // then this is probably their initial HEIGHT_V2 message // so they need a corresponding HEIGHT_V2 message from us - if (!peer.isOutbound() && (peer.getChainTipData() == null || peer.getChainTipData().getLastHeight() == null)) - peer.sendMessage(Network.getInstance().buildHeightMessage(peer, getChainTip())); + if (!peer.isOutbound() && peer.getChainTipData() == null) { + Message responseMessage = Network.getInstance().buildHeightOrChainTipInfo(peer); + + if (responseMessage == null || !peer.sendMessage(responseMessage)) { + peer.disconnect("failed to send our chain tip info"); + return; + } + } } // Update peer chain tip data - PeerChainTipData newChainTipData = new PeerChainTipData(heightV2Message.getHeight(), heightV2Message.getSignature(), heightV2Message.getTimestamp(), heightV2Message.getMinterPublicKey()); + BlockSummaryData newChainTipData = new BlockSummaryData(heightV2Message.getHeight(), heightV2Message.getSignature(), heightV2Message.getMinterPublicKey(), heightV2Message.getTimestamp()); peer.setChainTipData(newChainTipData); // Potentially synchronize Synchronizer.getInstance().requestSync(); } + private void onNetworkBlockSummariesV2Message(Peer peer, Message message) { + BlockSummariesV2Message blockSummariesV2Message = (BlockSummariesV2Message) message; + + if (!Settings.getInstance().isLite()) { + // If peer is inbound and we've not updated their height + // then this is probably their initial BLOCK_SUMMARIES_V2 message + // so they need a corresponding BLOCK_SUMMARIES_V2 message from us + if (!peer.isOutbound() && peer.getChainTipData() == null) { + Message responseMessage = Network.getInstance().buildHeightOrChainTipInfo(peer); + + if (responseMessage == null || !peer.sendMessage(responseMessage)) { + peer.disconnect("failed to send our chain tip info"); + return; + } + } + } + + // Update peer chain tip data + peer.setChainTipSummaries(blockSummariesV2Message.getBlockSummaries()); + + // Potentially synchronize + Synchronizer.getInstance().requestSync(); + } + private void onNetworkGetAccountMessage(Peer peer, Message message) { GetAccountMessage getAccountMessage = (GetAccountMessage) message; String address = getAccountMessage.getAddress(); @@ -1585,8 +1625,8 @@ public class Controller extends Thread { // Send valid, yet unexpected message type in response, so peer doesn't have to wait for timeout LOGGER.debug(() -> String.format("Sending 'account unknown' response to peer %s for GET_ACCOUNT request for unknown account %s", peer, address)); - // We'll send empty block summaries message as it's very short - Message accountUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + // Send generic 'unknown' message as it's very short + Message accountUnknownMessage = new GenericUnknownMessage(); accountUnknownMessage.setId(message.getId()); if (!peer.sendMessage(accountUnknownMessage)) peer.disconnect("failed to send account-unknown response"); @@ -1621,8 +1661,8 @@ public class Controller extends Thread { // Send valid, yet unexpected message type in response, so peer doesn't have to wait for timeout LOGGER.debug(() -> String.format("Sending 'account unknown' response to peer %s for GET_ACCOUNT_BALANCE request for unknown account %s and asset ID %d", peer, address, assetId)); - // We'll send empty block summaries message as it's very short - Message accountUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + // Send generic 'unknown' message as it's very short + Message accountUnknownMessage = new GenericUnknownMessage(); accountUnknownMessage.setId(message.getId()); if (!peer.sendMessage(accountUnknownMessage)) peer.disconnect("failed to send account-unknown response"); @@ -1665,8 +1705,8 @@ public class Controller extends Thread { // Send valid, yet unexpected message type in response, so peer doesn't have to wait for timeout LOGGER.debug(() -> String.format("Sending 'account unknown' response to peer %s for GET_ACCOUNT_TRANSACTIONS request for unknown account %s", peer, address)); - // We'll send empty block summaries message as it's very short - Message accountUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + // Send generic 'unknown' message as it's very short + Message accountUnknownMessage = new GenericUnknownMessage(); accountUnknownMessage.setId(message.getId()); if (!peer.sendMessage(accountUnknownMessage)) peer.disconnect("failed to send account-unknown response"); @@ -1702,8 +1742,8 @@ public class Controller extends Thread { // Send valid, yet unexpected message type in response, so peer doesn't have to wait for timeout LOGGER.debug(() -> String.format("Sending 'account unknown' response to peer %s for GET_ACCOUNT_NAMES request for unknown account %s", peer, address)); - // We'll send empty block summaries message as it's very short - Message accountUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + // Send generic 'unknown' message as it's very short + Message accountUnknownMessage = new GenericUnknownMessage(); accountUnknownMessage.setId(message.getId()); if (!peer.sendMessage(accountUnknownMessage)) peer.disconnect("failed to send account-unknown response"); @@ -1737,8 +1777,8 @@ public class Controller extends Thread { // Send valid, yet unexpected message type in response, so peer doesn't have to wait for timeout LOGGER.debug(() -> String.format("Sending 'name unknown' response to peer %s for GET_NAME request for unknown name %s", peer, name)); - // We'll send empty block summaries message as it's very short - Message nameUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + // Send generic 'unknown' message as it's very short + Message nameUnknownMessage = new GenericUnknownMessage(); nameUnknownMessage.setId(message.getId()); if (!peer.sendMessage(nameUnknownMessage)) peer.disconnect("failed to send name-unknown response"); @@ -1786,14 +1826,14 @@ public class Controller extends Thread { continue; } - final PeerChainTipData peerChainTipData = peer.getChainTipData(); + BlockSummaryData peerChainTipData = peer.getChainTipData(); if (peerChainTipData == null) { iterator.remove(); continue; } // Disregard peers that don't have a recent block - if (peerChainTipData.getLastBlockTimestamp() == null || peerChainTipData.getLastBlockTimestamp() < minLatestBlockTimestamp) { + if (peerChainTipData.getTimestamp() == null || peerChainTipData.getTimestamp() < minLatestBlockTimestamp) { iterator.remove(); continue; } diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 74a4a785..a6fbfe71 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -19,7 +19,6 @@ import org.qortal.block.BlockChain; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.block.CommonBlockData; -import org.qortal.data.network.PeerChainTipData; import org.qortal.data.transaction.RewardShareTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.event.Event; @@ -282,7 +281,7 @@ public class Synchronizer extends Thread { BlockData priorChainTip = Controller.getInstance().getChainTip(); synchronized (this.syncLock) { - this.syncPercent = (priorChainTip.getHeight() * 100) / peer.getChainTipData().getLastHeight(); + this.syncPercent = (priorChainTip.getHeight() * 100) / peer.getChainTipData().getHeight(); // Only update SysTray if we're potentially changing height if (this.syncPercent < 100) { @@ -312,7 +311,7 @@ public class Synchronizer extends Thread { case INFERIOR_CHAIN: { // Update our list of inferior chain tips - ByteArray inferiorChainSignature = ByteArray.wrap(peer.getChainTipData().getLastBlockSignature()); + ByteArray inferiorChainSignature = ByteArray.wrap(peer.getChainTipData().getSignature()); if (!inferiorChainSignatures.contains(inferiorChainSignature)) inferiorChainSignatures.add(inferiorChainSignature); @@ -320,7 +319,8 @@ public class Synchronizer extends Thread { LOGGER.debug(() -> String.format("Refused to synchronize with peer %s (%s)", peer, syncResult.name())); // Notify peer of our superior chain - if (!peer.sendMessage(Network.getInstance().buildHeightMessage(peer, priorChainTip))) + Message message = Network.getInstance().buildHeightOrChainTipInfo(peer); + if (message == null || !peer.sendMessage(message)) peer.disconnect("failed to notify peer of our superior chain"); break; } @@ -341,7 +341,7 @@ public class Synchronizer extends Thread { // fall-through... case NOTHING_TO_DO: { // Update our list of inferior chain tips - ByteArray inferiorChainSignature = ByteArray.wrap(peer.getChainTipData().getLastBlockSignature()); + ByteArray inferiorChainSignature = ByteArray.wrap(peer.getChainTipData().getSignature()); if (!inferiorChainSignatures.contains(inferiorChainSignature)) inferiorChainSignatures.add(inferiorChainSignature); @@ -369,8 +369,7 @@ public class Synchronizer extends Thread { // Reset our cache of inferior chains inferiorChainSignatures.clear(); - Network network = Network.getInstance(); - network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip)); + Network.getInstance().broadcastOurChain(); EventBus.INSTANCE.notify(new NewChainTipEvent(priorChainTip, newChainTip)); } @@ -513,13 +512,13 @@ public class Synchronizer extends Thread { final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); final int ourInitialHeight = ourLatestBlockData.getHeight(); - PeerChainTipData peerChainTipData = peer.getChainTipData(); - int peerHeight = peerChainTipData.getLastHeight(); - byte[] peersLastBlockSignature = peerChainTipData.getLastBlockSignature(); + BlockSummaryData peerChainTipData = peer.getChainTipData(); + int peerHeight = peerChainTipData.getHeight(); + byte[] peersLastBlockSignature = peerChainTipData.getSignature(); byte[] ourLastBlockSignature = ourLatestBlockData.getSignature(); LOGGER.debug(String.format("Fetching summaries from peer %s at height %d, sig %.8s, ts %d; our height %d, sig %.8s, ts %d", peer, - peerHeight, Base58.encode(peersLastBlockSignature), peer.getChainTipData().getLastBlockTimestamp(), + peerHeight, Base58.encode(peersLastBlockSignature), peerChainTipData.getTimestamp(), ourInitialHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp())); List peerBlockSummaries = new ArrayList<>(); @@ -637,9 +636,9 @@ public class Synchronizer extends Thread { return peers; // Count the number of blocks this peer has beyond our common block - final PeerChainTipData peerChainTipData = peer.getChainTipData(); - final int peerHeight = peerChainTipData.getLastHeight(); - final byte[] peerLastBlockSignature = peerChainTipData.getLastBlockSignature(); + final BlockSummaryData peerChainTipData = peer.getChainTipData(); + final int peerHeight = peerChainTipData.getHeight(); + final byte[] peerLastBlockSignature = peerChainTipData.getSignature(); final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); // Limit the number of blocks we are comparing. FUTURE: we could request more in batches, but there may not be a case when this is needed int summariesRequired = Math.min(peerAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE); @@ -727,8 +726,9 @@ public class Synchronizer extends Thread { LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature()))); for (Peer peer : peersSharingCommonBlock) { - final int peerHeight = peer.getChainTipData().getLastHeight(); - final Long peerLastBlockTimestamp = peer.getChainTipData().getLastBlockTimestamp(); + BlockSummaryData peerChainTipData = peer.getChainTipData(); + final int peerHeight = peerChainTipData.getHeight(); + final Long peerLastBlockTimestamp = peerChainTipData.getTimestamp(); final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); final CommonBlockData peerCommonBlockData = peer.getCommonBlockData(); @@ -825,7 +825,7 @@ public class Synchronizer extends Thread { // Calculate the length of the shortest peer chain sharing this common block int minChainLength = 0; for (Peer peer : peersSharingCommonBlock) { - final int peerHeight = peer.getChainTipData().getLastHeight(); + final int peerHeight = peer.getChainTipData().getHeight(); final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); if (peerAdditionalBlocksAfterCommonBlock < minChainLength || minChainLength == 0) @@ -933,13 +933,13 @@ public class Synchronizer extends Thread { final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); final int ourInitialHeight = ourLatestBlockData.getHeight(); - PeerChainTipData peerChainTipData = peer.getChainTipData(); - int peerHeight = peerChainTipData.getLastHeight(); - byte[] peersLastBlockSignature = peerChainTipData.getLastBlockSignature(); + BlockSummaryData peerChainTipData = peer.getChainTipData(); + int peerHeight = peerChainTipData.getHeight(); + byte[] peersLastBlockSignature = peerChainTipData.getSignature(); byte[] ourLastBlockSignature = ourLatestBlockData.getSignature(); String syncString = String.format("Synchronizing with peer %s at height %d, sig %.8s, ts %d; our height %d, sig %.8s, ts %d", peer, - peerHeight, Base58.encode(peersLastBlockSignature), peer.getChainTipData().getLastBlockTimestamp(), + peerHeight, Base58.encode(peersLastBlockSignature), peerChainTipData.getTimestamp(), ourInitialHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp()); LOGGER.info(syncString); @@ -1313,7 +1313,7 @@ public class Synchronizer extends Thread { // Final check to make sure the peer isn't out of date (except for when we're in recovery mode) if (!recoveryMode && peer.getChainTipData() != null) { final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); - final Long peerLastBlockTimestamp = peer.getChainTipData().getLastBlockTimestamp(); + final Long peerLastBlockTimestamp = peer.getChainTipData().getTimestamp(); if (peerLastBlockTimestamp == null || peerLastBlockTimestamp < minLatestBlockTimestamp) { LOGGER.info(String.format("Peer %s is out of date, so abandoning sync attempt", peer)); return SynchronizationResult.CHAIN_TIP_TOO_OLD; diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 22cf4144..30b0fcca 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -595,9 +595,10 @@ public class ArbitraryDataFileManager extends Thread { // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout LOGGER.debug(String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile)); - // We'll send empty block summaries message as it's very short - // TODO: use a different message type here - Message fileUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + // Send generic 'unknown' message as it's very short + Message fileUnknownMessage = peer.getPeersVersion() >= GenericUnknownMessage.MINIMUM_PEER_VERSION + ? new GenericUnknownMessage() + : new BlockSummariesMessage(Collections.emptyList()); fileUnknownMessage.setId(message.getId()); if (!peer.sendMessage(fileUnknownMessage)) { LOGGER.debug("Couldn't sent file-unknown response"); diff --git a/src/main/java/org/qortal/data/block/BlockSummaryData.java b/src/main/java/org/qortal/data/block/BlockSummaryData.java index 2167f0f0..57e29d0d 100644 --- a/src/main/java/org/qortal/data/block/BlockSummaryData.java +++ b/src/main/java/org/qortal/data/block/BlockSummaryData.java @@ -11,11 +11,12 @@ public class BlockSummaryData { private int height; private byte[] signature; private byte[] minterPublicKey; - private int onlineAccountsCount; // Optional, set during construction + private Integer onlineAccountsCount; private Long timestamp; private Integer transactionCount; + private byte[] reference; // Optional, set after construction private Integer minterLevel; @@ -25,6 +26,15 @@ public class BlockSummaryData { protected BlockSummaryData() { } + /** Constructor typically populated with fields from HeightV2Message */ + public BlockSummaryData(int height, byte[] signature, byte[] minterPublicKey, long timestamp) { + this.height = height; + this.signature = signature; + this.minterPublicKey = minterPublicKey; + this.timestamp = timestamp; + } + + /** Constructor typically populated with fields from BlockSummariesMessage */ public BlockSummaryData(int height, byte[] signature, byte[] minterPublicKey, int onlineAccountsCount) { this.height = height; this.signature = signature; @@ -32,13 +42,16 @@ public class BlockSummaryData { this.onlineAccountsCount = onlineAccountsCount; } - public BlockSummaryData(int height, byte[] signature, byte[] minterPublicKey, int onlineAccountsCount, long timestamp, int transactionCount) { + /** Constructor typically populated with fields from BlockSummariesV2Message */ + public BlockSummaryData(int height, byte[] signature, byte[] minterPublicKey, Integer onlineAccountsCount, + Long timestamp, Integer transactionCount, byte[] reference) { this.height = height; this.signature = signature; this.minterPublicKey = minterPublicKey; this.onlineAccountsCount = onlineAccountsCount; this.timestamp = timestamp; this.transactionCount = transactionCount; + this.reference = reference; } public BlockSummaryData(BlockData blockData) { @@ -49,6 +62,7 @@ public class BlockSummaryData { this.timestamp = blockData.getTimestamp(); this.transactionCount = blockData.getTransactionCount(); + this.reference = blockData.getReference(); } // Getters / setters @@ -65,7 +79,7 @@ public class BlockSummaryData { return this.minterPublicKey; } - public int getOnlineAccountsCount() { + public Integer getOnlineAccountsCount() { return this.onlineAccountsCount; } @@ -77,6 +91,10 @@ public class BlockSummaryData { return this.transactionCount; } + public byte[] getReference() { + return this.reference; + } + public Integer getMinterLevel() { return this.minterLevel; } diff --git a/src/main/java/org/qortal/data/block/CommonBlockData.java b/src/main/java/org/qortal/data/block/CommonBlockData.java index dd502df7..37e9649b 100644 --- a/src/main/java/org/qortal/data/block/CommonBlockData.java +++ b/src/main/java/org/qortal/data/block/CommonBlockData.java @@ -1,7 +1,5 @@ package org.qortal.data.block; -import org.qortal.data.network.PeerChainTipData; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import java.math.BigInteger; @@ -14,14 +12,14 @@ public class CommonBlockData { private BlockSummaryData commonBlockSummary = null; private List blockSummariesAfterCommonBlock = null; private BigInteger chainWeight = null; - private PeerChainTipData chainTipData = null; + private BlockSummaryData chainTipData = null; // Constructors protected CommonBlockData() { } - public CommonBlockData(BlockSummaryData commonBlockSummary, PeerChainTipData chainTipData) { + public CommonBlockData(BlockSummaryData commonBlockSummary, BlockSummaryData chainTipData) { this.commonBlockSummary = commonBlockSummary; this.chainTipData = chainTipData; } @@ -49,7 +47,7 @@ public class CommonBlockData { this.chainWeight = chainWeight; } - public PeerChainTipData getChainTipData() { + public BlockSummaryData getChainTipData() { return this.chainTipData; } diff --git a/src/main/java/org/qortal/data/network/PeerChainTipData.java b/src/main/java/org/qortal/data/network/PeerChainTipData.java deleted file mode 100644 index d8dbbad4..00000000 --- a/src/main/java/org/qortal/data/network/PeerChainTipData.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.qortal.data.network; - -public class PeerChainTipData { - - /** Latest block height as reported by peer. */ - private Integer lastHeight; - /** Latest block signature as reported by peer. */ - private byte[] lastBlockSignature; - /** Latest block timestamp as reported by peer. */ - private Long lastBlockTimestamp; - /** Latest block minter public key as reported by peer. */ - private byte[] lastBlockMinter; - - public PeerChainTipData(Integer lastHeight, byte[] lastBlockSignature, Long lastBlockTimestamp, byte[] lastBlockMinter) { - this.lastHeight = lastHeight; - this.lastBlockSignature = lastBlockSignature; - this.lastBlockTimestamp = lastBlockTimestamp; - this.lastBlockMinter = lastBlockMinter; - } - - public Integer getLastHeight() { - return this.lastHeight; - } - - public byte[] getLastBlockSignature() { - return this.lastBlockSignature; - } - - public Long getLastBlockTimestamp() { - return this.lastBlockTimestamp; - } - - public byte[] getLastBlockMinter() { - return this.lastBlockMinter; - } - -} diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 57073e99..8aac68f0 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -11,6 +11,7 @@ import org.qortal.controller.arbitrary.ArbitraryDataFileListManager; import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.crypto.Crypto; import org.qortal.data.block.BlockData; +import org.qortal.data.block.BlockSummaryData; import org.qortal.data.network.PeerData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.message.*; @@ -90,6 +91,8 @@ public class Network { private static final long DISCONNECTION_CHECK_INTERVAL = 10 * 1000L; // milliseconds + private static final int BROADCAST_CHAIN_TIP_DEPTH = 7; // Just enough to fill a SINGLE TCP packet (~1440 bytes) + // Generate our node keys / ID private final Ed25519PrivateKeyParameters edPrivateKeyParams = new Ed25519PrivateKeyParameters(new SecureRandom()); private final Ed25519PublicKeyParameters edPublicKeyParams = edPrivateKeyParams.generatePublicKey(); @@ -1087,10 +1090,16 @@ public class Network { if (peer.isOutbound()) { if (!Settings.getInstance().isLite()) { - // Send our height - Message heightMessage = buildHeightMessage(peer, Controller.getInstance().getChainTip()); - if (!peer.sendMessage(heightMessage)) { - peer.disconnect("failed to send height/info"); + // Send our height / chain tip info + Message message = this.buildHeightOrChainTipInfo(peer); + + if (message == null) { + peer.disconnect("Couldn't build our chain tip info"); + return; + } + + if (!peer.sendMessage(message)) { + peer.disconnect("failed to send height / chain tip info"); return; } } @@ -1164,10 +1173,47 @@ public class Network { return new PeersV2Message(peerAddresses); } - public Message buildHeightMessage(Peer peer, BlockData blockData) { - // HEIGHT_V2 contains way more useful info - return new HeightV2Message(blockData.getHeight(), blockData.getSignature(), - blockData.getTimestamp(), blockData.getMinterPublicKey()); + /** Builds either (legacy) HeightV2Message or (newer) BlockSummariesV2Message, depending on peer version. + * + * @return Message, or null if DataException was thrown. + */ + public Message buildHeightOrChainTipInfo(Peer peer) { + if (peer.getPeersVersion() >= BlockSummariesV2Message.MINIMUM_PEER_VERSION) { + int latestHeight = Controller.getInstance().getChainHeight(); + + try (final Repository repository = RepositoryManager.getRepository()) { + List latestBlockSummaries = repository.getBlockRepository().getBlockSummaries(latestHeight - BROADCAST_CHAIN_TIP_DEPTH, latestHeight); + return new BlockSummariesV2Message(latestBlockSummaries); + } catch (DataException e) { + return null; + } + } else { + // For older peers + BlockData latestBlockData = Controller.getInstance().getChainTip(); + return new HeightV2Message(latestBlockData.getHeight(), latestBlockData.getSignature(), + latestBlockData.getTimestamp(), latestBlockData.getMinterPublicKey()); + } + } + + public void broadcastOurChain() { + BlockData latestBlockData = Controller.getInstance().getChainTip(); + int latestHeight = latestBlockData.getHeight(); + + try (final Repository repository = RepositoryManager.getRepository()) { + List latestBlockSummaries = repository.getBlockRepository().getBlockSummaries(latestHeight - BROADCAST_CHAIN_TIP_DEPTH, latestHeight); + Message latestBlockSummariesMessage = new BlockSummariesV2Message(latestBlockSummaries); + + // For older peers + Message heightMessage = new HeightV2Message(latestBlockData.getHeight(), latestBlockData.getSignature(), + latestBlockData.getTimestamp(), latestBlockData.getMinterPublicKey()); + + Network.getInstance().broadcast(broadcastPeer -> broadcastPeer.getPeersVersion() >= BlockSummariesV2Message.MINIMUM_PEER_VERSION + ? latestBlockSummariesMessage + : heightMessage + ); + } catch (DataException e) { + LOGGER.warn("Couldn't broadcast our chain tip info", e); + } } public Message buildNewTransactionMessage(Peer peer, TransactionData transactionData) { diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index cac0ccc9..a187d29b 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -6,8 +6,8 @@ import com.google.common.net.InetAddresses; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; +import org.qortal.data.block.BlockSummaryData; import org.qortal.data.block.CommonBlockData; -import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; import org.qortal.network.message.ChallengeMessage; import org.qortal.network.message.Message; @@ -148,7 +148,7 @@ public class Peer { /** * Latest block info as reported by peer. */ - private PeerChainTipData peersChainTipData; + private List peersChainTipData = Collections.emptyList(); /** * Our common block with this peer @@ -353,28 +353,34 @@ public class Peer { } } - public PeerChainTipData getChainTipData() { - synchronized (this.peerInfoLock) { - return this.peersChainTipData; - } + public BlockSummaryData getChainTipData() { + List chainTipSummaries = this.peersChainTipData; + + if (chainTipSummaries.isEmpty()) + return null; + + // Return last entry, which should have greatest height + return chainTipSummaries.get(chainTipSummaries.size() - 1); } - public void setChainTipData(PeerChainTipData chainTipData) { - synchronized (this.peerInfoLock) { - this.peersChainTipData = chainTipData; - } + public void setChainTipData(BlockSummaryData chainTipData) { + this.peersChainTipData = Collections.singletonList(chainTipData); + } + + public List getChainTipSummaries() { + return this.peersChainTipData; + } + + public void setChainTipSummaries(List chainTipSummaries) { + this.peersChainTipData = List.copyOf(chainTipSummaries); } public CommonBlockData getCommonBlockData() { - synchronized (this.peerInfoLock) { - return this.commonBlockData; - } + return this.commonBlockData; } public void setCommonBlockData(CommonBlockData commonBlockData) { - synchronized (this.peerInfoLock) { - this.commonBlockData = commonBlockData; - } + this.commonBlockData = commonBlockData; } public boolean isSyncInProgress() { @@ -904,20 +910,22 @@ public class Peer { // Common block data public boolean canUseCachedCommonBlockData() { - PeerChainTipData peerChainTipData = this.getChainTipData(); - CommonBlockData commonBlockData = this.getCommonBlockData(); + BlockSummaryData peerChainTipData = this.getChainTipData(); + if (peerChainTipData == null || peerChainTipData.getSignature() == null) + return false; - if (peerChainTipData != null && commonBlockData != null) { - PeerChainTipData commonBlockChainTipData = commonBlockData.getChainTipData(); - if (peerChainTipData.getLastBlockSignature() != null && commonBlockChainTipData != null - && commonBlockChainTipData.getLastBlockSignature() != null) { - if (Arrays.equals(peerChainTipData.getLastBlockSignature(), - commonBlockChainTipData.getLastBlockSignature())) { - return true; - } - } - } - return false; + CommonBlockData commonBlockData = this.getCommonBlockData(); + if (commonBlockData == null) + return false; + + BlockSummaryData commonBlockChainTipData = commonBlockData.getChainTipData(); + if (commonBlockChainTipData == null || commonBlockChainTipData.getSignature() == null) + return false; + + if (!Arrays.equals(peerChainTipData.getSignature(), commonBlockChainTipData.getSignature())) + return false; + + return true; } diff --git a/src/main/java/org/qortal/network/message/BlockSummariesV2Message.java b/src/main/java/org/qortal/network/message/BlockSummariesV2Message.java new file mode 100644 index 00000000..96c661a4 --- /dev/null +++ b/src/main/java/org/qortal/network/message/BlockSummariesV2Message.java @@ -0,0 +1,104 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.qortal.data.block.BlockSummaryData; +import org.qortal.transform.Transformer; +import org.qortal.transform.block.BlockTransformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class BlockSummariesV2Message extends Message { + + public static final long MINIMUM_PEER_VERSION = 0x03000400cbL; + + private static final int BLOCK_SUMMARY_V2_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH /* block signature */ + + Transformer.PUBLIC_KEY_LENGTH /* minter public key */ + + Transformer.INT_LENGTH /* online accounts count */ + + Transformer.LONG_LENGTH /* block timestamp */ + + Transformer.INT_LENGTH /* transactions count */ + + BlockTransformer.BLOCK_SIGNATURE_LENGTH; /* block reference */ + + private List blockSummaries; + + public BlockSummariesV2Message(List blockSummaries) { + super(MessageType.BLOCK_SUMMARIES_V2); + + // Shortcut for when there are no summaries + if (blockSummaries.isEmpty()) { + this.dataBytes = Message.EMPTY_DATA_BYTES; + return; + } + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + // First summary's height + bytes.write(Ints.toByteArray(blockSummaries.get(0).getHeight())); + + for (BlockSummaryData blockSummary : blockSummaries) { + bytes.write(blockSummary.getSignature()); + bytes.write(blockSummary.getMinterPublicKey()); + bytes.write(Ints.toByteArray(blockSummary.getOnlineAccountsCount())); + bytes.write(Longs.toByteArray(blockSummary.getTimestamp())); + bytes.write(Ints.toByteArray(blockSummary.getTransactionCount())); + bytes.write(blockSummary.getReference()); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private BlockSummariesV2Message(int id, List blockSummaries) { + super(id, MessageType.BLOCK_SUMMARIES_V2); + + this.blockSummaries = blockSummaries; + } + + public List getBlockSummaries() { + return this.blockSummaries; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + int height = bytes.getInt(); + + // Expecting bytes remaining to be exact multiples of BLOCK_SUMMARY_V2_LENGTH + if (bytes.remaining() % BLOCK_SUMMARY_V2_LENGTH != 0) + throw new BufferUnderflowException(); + + List blockSummaries = new ArrayList<>(); + while (bytes.hasRemaining()) { + byte[] signature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; + bytes.get(signature); + + byte[] minterPublicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + bytes.get(minterPublicKey); + + int onlineAccountsCount = bytes.getInt(); + + long timestamp = bytes.getLong(); + + int transactionsCount = bytes.getInt(); + + byte[] reference = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; + bytes.get(reference); + + BlockSummaryData blockSummary = new BlockSummaryData(height, signature, minterPublicKey, + onlineAccountsCount, timestamp, transactionsCount, reference); + blockSummaries.add(blockSummary); + + height++; + } + + return new BlockSummariesV2Message(id, blockSummaries); + } + +} diff --git a/src/main/java/org/qortal/network/message/GenericUnknownMessage.java b/src/main/java/org/qortal/network/message/GenericUnknownMessage.java new file mode 100644 index 00000000..15faaa1b --- /dev/null +++ b/src/main/java/org/qortal/network/message/GenericUnknownMessage.java @@ -0,0 +1,23 @@ +package org.qortal.network.message; + +import java.nio.ByteBuffer; + +public class GenericUnknownMessage extends Message { + + public static final long MINIMUM_PEER_VERSION = 0x03000400cbL; + + public GenericUnknownMessage() { + super(MessageType.GENERIC_UNKNOWN); + + this.dataBytes = EMPTY_DATA_BYTES; + } + + private GenericUnknownMessage(int id) { + super(id, MessageType.GENERIC_UNKNOWN); + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + return new GenericUnknownMessage(id); + } + +} diff --git a/src/main/java/org/qortal/network/message/MessageType.java b/src/main/java/org/qortal/network/message/MessageType.java index 087e7fbf..4dd4a3c8 100644 --- a/src/main/java/org/qortal/network/message/MessageType.java +++ b/src/main/java/org/qortal/network/message/MessageType.java @@ -21,6 +21,7 @@ public enum MessageType { HEIGHT_V2(10, HeightV2Message::fromByteBuffer), PING(11, PingMessage::fromByteBuffer), PONG(12, PongMessage::fromByteBuffer), + GENERIC_UNKNOWN(13, GenericUnknownMessage::fromByteBuffer), // Requesting data PEERS_V2(20, PeersV2Message::fromByteBuffer), @@ -41,6 +42,7 @@ public enum MessageType { BLOCK_SUMMARIES(70, BlockSummariesMessage::fromByteBuffer), GET_BLOCK_SUMMARIES(71, GetBlockSummariesMessage::fromByteBuffer), + BLOCK_SUMMARIES_V2(72, BlockSummariesV2Message::fromByteBuffer), ONLINE_ACCOUNTS(80, OnlineAccountsMessage::fromByteBuffer), GET_ONLINE_ACCOUNTS(81, GetOnlineAccountsMessage::fromByteBuffer), diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java index cc7e1611..c3c5638a 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java @@ -143,13 +143,17 @@ public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository { byte[] blockMinterPublicKey = resultSet.getBytes(3); // Fetch additional info from the archive itself - int onlineAccountsCount = 0; + Integer onlineAccountsCount = null; + Long timestamp = null; + Integer transactionCount = null; + byte[] reference = null; + BlockData blockData = this.fromSignature(signature); if (blockData != null) { onlineAccountsCount = blockData.getOnlineAccountsCount(); } - BlockSummaryData blockSummary = new BlockSummaryData(height, signature, blockMinterPublicKey, onlineAccountsCount); + BlockSummaryData blockSummary = new BlockSummaryData(height, signature, blockMinterPublicKey, onlineAccountsCount, timestamp, transactionCount, reference); blockSummaries.add(blockSummary); } while (resultSet.next()); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java index b8238085..f38d549c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java @@ -297,7 +297,7 @@ public class HSQLDBBlockRepository implements BlockRepository { @Override public List getBlockSummariesBySigner(byte[] signerPublicKey, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(512); - sql.append("SELECT signature, height, Blocks.minter, online_accounts_count FROM "); + sql.append("SELECT signature, height, Blocks.minter, online_accounts_count, minted_when, transaction_count, Blocks.reference FROM "); // List of minter account's public key and reward-share public keys with minter's public key sql.append("(SELECT * FROM (VALUES (CAST(? AS QortalPublicKey))) UNION (SELECT reward_share_public_key FROM RewardShares WHERE minter_public_key = ?)) AS PublicKeys (public_key) "); @@ -322,8 +322,12 @@ public class HSQLDBBlockRepository implements BlockRepository { int height = resultSet.getInt(2); byte[] blockMinterPublicKey = resultSet.getBytes(3); int onlineAccountsCount = resultSet.getInt(4); + long timestamp = resultSet.getLong(5); + int transactionCount = resultSet.getInt(6); + byte[] reference = resultSet.getBytes(7); - BlockSummaryData blockSummary = new BlockSummaryData(height, signature, blockMinterPublicKey, onlineAccountsCount); + BlockSummaryData blockSummary = new BlockSummaryData(height, signature, blockMinterPublicKey, onlineAccountsCount, + timestamp, transactionCount, reference); blockSummaries.add(blockSummary); } while (resultSet.next()); @@ -355,7 +359,7 @@ public class HSQLDBBlockRepository implements BlockRepository { @Override public List getBlockSummaries(int firstBlockHeight, int lastBlockHeight) throws DataException { - String sql = "SELECT signature, height, minter, online_accounts_count, minted_when, transaction_count " + String sql = "SELECT signature, height, minter, online_accounts_count, minted_when, transaction_count, reference " + "FROM Blocks WHERE height BETWEEN ? AND ?"; List blockSummaries = new ArrayList<>(); @@ -371,9 +375,10 @@ public class HSQLDBBlockRepository implements BlockRepository { int onlineAccountsCount = resultSet.getInt(4); long timestamp = resultSet.getLong(5); int transactionCount = resultSet.getInt(6); + byte[] reference = resultSet.getBytes(7); BlockSummaryData blockSummary = new BlockSummaryData(height, signature, minterPublicKey, onlineAccountsCount, - timestamp, transactionCount); + timestamp, transactionCount, reference); blockSummaries.add(blockSummary); } while (resultSet.next()); From e80dd31fb4a9ccfb960218561fcc242234801bb2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Sep 2022 13:53:27 +0100 Subject: [PATCH 55/83] BlockSummariesV2Message.MINIMUM_PEER_VERSION set to 3.6.1 --- .../org/qortal/network/message/BlockSummariesV2Message.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/message/BlockSummariesV2Message.java b/src/main/java/org/qortal/network/message/BlockSummariesV2Message.java index 96c661a4..6ed6c8aa 100644 --- a/src/main/java/org/qortal/network/message/BlockSummariesV2Message.java +++ b/src/main/java/org/qortal/network/message/BlockSummariesV2Message.java @@ -15,7 +15,7 @@ import java.util.List; public class BlockSummariesV2Message extends Message { - public static final long MINIMUM_PEER_VERSION = 0x03000400cbL; + public static final long MINIMUM_PEER_VERSION = 0x0300060001L; private static final int BLOCK_SUMMARY_V2_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH /* block signature */ + Transformer.PUBLIC_KEY_LENGTH /* minter public key */ From 7a60f713ead6bded0062a026af4cc147cb1a952d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Sep 2022 14:35:02 +0100 Subject: [PATCH 56/83] Fixed error in rebase. --- src/main/java/org/qortal/controller/BlockMinter.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index a07d37fe..100e74db 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -434,9 +434,14 @@ public class BlockMinter extends Thread { blockchainLock.unlock(); } - if (newBlockMinted) { - // Broadcast our new chain to network - Network.getInstance().broadcastOurChain(); + if (newBlockMinted) { + // Broadcast our new chain to network + Network.getInstance().broadcastOurChain(); + } + + } catch (InterruptedException e) { + // We've been interrupted - time to exit + return; } } } catch (DataException e) { From d2ebb215e605709a498170d100d0bc57a4b01ed4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Sep 2022 14:36:49 +0100 Subject: [PATCH 57/83] Fixed Synchronizer.getBlockSummaries() which was expecting BLOCK_SUMMARIES, but updated peers send BLOCK_SUMMARIES_V2 --- .../java/org/qortal/controller/Synchronizer.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index a6fbfe71..a8d91f52 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -1553,12 +1553,19 @@ public class Synchronizer extends Thread { Message getBlockSummariesMessage = new GetBlockSummariesMessage(parentSignature, numberRequested); Message message = peer.getResponse(getBlockSummariesMessage); - if (message == null || message.getType() != MessageType.BLOCK_SUMMARIES) + if (message == null) return null; - BlockSummariesMessage blockSummariesMessage = (BlockSummariesMessage) message; + if (message.getType() == MessageType.BLOCK_SUMMARIES) { + BlockSummariesMessage blockSummariesMessage = (BlockSummariesMessage) message; + return blockSummariesMessage.getBlockSummaries(); + } + else if (message.getType() == MessageType.BLOCK_SUMMARIES_V2) { + BlockSummariesV2Message blockSummariesMessage = (BlockSummariesV2Message) message; + return blockSummariesMessage.getBlockSummaries(); + } - return blockSummariesMessage.getBlockSummaries(); + return null; } private List getBlockSignatures(Peer peer, byte[] parentSignature, int numberRequested) throws InterruptedException { From 309f27a6b83a6745583a887b5150d790a43bdcf3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Sep 2022 15:21:01 +0100 Subject: [PATCH 58/83] Moved error to debug, as we now get a burst of these soon after startup, due to commit 99858f3. This also shows that commit 99858f3 now prevents a block candidate with a very small number of online accounts being built immediately after startup. --- src/main/java/org/qortal/block/Block.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index bdae83c2..07c7db6f 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -375,7 +375,7 @@ public class Block { } if (onlineAccounts.isEmpty()) { - LOGGER.error("No online accounts - not even our own?"); + LOGGER.debug("No online accounts - not even our own?"); return null; } From 5c746f0bd90fc5aa8de6a5800608bd21e84164d9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Sep 2022 15:48:45 +0100 Subject: [PATCH 59/83] Fixed bug which required a node to hold local trade presences before it would request any. This caused large gaps with no presence data. They are removed when they expire, causing the local count to drop to zero, and the node would only start requesting them again once a peer had pushed one or more entries proactively. --- src/main/java/org/qortal/controller/tradebot/TradeBot.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index c7ae1db3..85e594fa 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -468,9 +468,6 @@ public class TradeBot implements Listener { List safeTradePresences = List.copyOf(this.safeAllTradePresencesByPubkey.values()); - if (safeTradePresences.isEmpty()) - return; - LOGGER.debug("Broadcasting all {} known trade presences. Next broadcast timestamp: {}", safeTradePresences.size(), nextTradePresenceBroadcastTimestamp ); From 4681218416c86e841dc2afa58436c9166ab46bae Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Sep 2022 15:49:29 +0100 Subject: [PATCH 60/83] Include total count in debug trade presence logging --- src/main/java/org/qortal/controller/tradebot/TradeBot.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index 85e594fa..5880f561 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -634,7 +634,7 @@ public class TradeBot implements Listener { } if (newCount > 0) { - LOGGER.debug("New trade presences: {}", newCount); + LOGGER.debug("New trade presences: {}, all trade presences: {}", newCount, allTradePresencesByPubkey.size()); rebuildSafeAllTradePresences(); } } From aa9da45c01657e218a9f249497cc8c25e164d0f5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 25 Sep 2022 11:37:07 +0100 Subject: [PATCH 61/83] Added optional filtering by reference in GET /chat/messages --- src/main/java/org/qortal/api/resource/ChatResource.java | 6 ++++++ .../org/qortal/api/websocket/ChatMessagesWebSocket.java | 2 ++ src/main/java/org/qortal/repository/ChatRepository.java | 2 +- .../org/qortal/repository/hsqldb/HSQLDBChatRepository.java | 7 ++++++- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ChatResource.java b/src/main/java/org/qortal/api/resource/ChatResource.java index 0bbd1951..ee2a8599 100644 --- a/src/main/java/org/qortal/api/resource/ChatResource.java +++ b/src/main/java/org/qortal/api/resource/ChatResource.java @@ -69,6 +69,7 @@ public class ChatResource { public List searchChat(@QueryParam("before") Long before, @QueryParam("after") Long after, @QueryParam("txGroupId") Integer txGroupId, @QueryParam("involving") List involvingAddresses, + @QueryParam("reference") String reference, @Parameter(ref = "limit") @QueryParam("limit") Integer limit, @Parameter(ref = "offset") @QueryParam("offset") Integer offset, @Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) { @@ -87,11 +88,16 @@ public class ChatResource { if (after != null && after < 1500000000000L) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + byte[] referenceBytes = null; + if (reference != null) + referenceBytes = Base58.decode(reference); + try (final Repository repository = RepositoryManager.getRepository()) { return repository.getChatRepository().getMessagesMatchingCriteria( before, after, txGroupId, + referenceBytes, involvingAddresses, limit, offset, reverse); } catch (DataException e) { diff --git a/src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java b/src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java index 3dc2d494..9760b7f0 100644 --- a/src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java @@ -46,6 +46,7 @@ public class ChatMessagesWebSocket extends ApiWebSocket { null, txGroupId, null, + null, null, null, null); sendMessages(session, chatMessages); @@ -72,6 +73,7 @@ public class ChatMessagesWebSocket extends ApiWebSocket { null, null, null, + null, involvingAddresses, null, null, null); diff --git a/src/main/java/org/qortal/repository/ChatRepository.java b/src/main/java/org/qortal/repository/ChatRepository.java index cd4b9a8f..2ecd8a34 100644 --- a/src/main/java/org/qortal/repository/ChatRepository.java +++ b/src/main/java/org/qortal/repository/ChatRepository.java @@ -14,7 +14,7 @@ public interface ChatRepository { * Expects EITHER non-null txGroupID OR non-null sender and recipient addresses. */ public List getMessagesMatchingCriteria(Long before, Long after, - Integer txGroupId, List involving, + Integer txGroupId, byte[] reference, List involving, Integer limit, Integer offset, Boolean reverse) throws DataException; public ChatMessage toChatMessage(ChatTransactionData chatTransactionData) throws DataException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java index 2972e9f2..2f570686 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java @@ -23,7 +23,7 @@ public class HSQLDBChatRepository implements ChatRepository { } @Override - public List getMessagesMatchingCriteria(Long before, Long after, Integer txGroupId, + public List getMessagesMatchingCriteria(Long before, Long after, Integer txGroupId, byte[] referenceBytes, List involving, Integer limit, Integer offset, Boolean reverse) throws DataException { // Check args meet expectations @@ -57,6 +57,11 @@ public class HSQLDBChatRepository implements ChatRepository { bindParams.add(after); } + if (referenceBytes != null) { + whereClauses.add("reference = ?"); + bindParams.add(referenceBytes); + } + if (txGroupId != null) { whereClauses.add("tx_group_id = " + txGroupId); // int safe to use literally whereClauses.add("recipient IS NULL"); From 5989473c8a9bd2c09134e46a262878ef8b286ad3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 25 Sep 2022 12:06:14 +0100 Subject: [PATCH 62/83] Revert "Allow duplicate variations of each OnlineAccountData in the import queue, but don't allow two entries that match exactly." This reverts commit 6d9e6e8d4c89582ffad23c1094b6a8e3aee91116. --- .../controller/OnlineAccountsManager.java | 24 ++----------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 40192876..47d8cf1b 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -71,7 +71,7 @@ public class OnlineAccountsManager { private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(4, new NamedThreadFactory("OnlineAccounts")); private volatile boolean isStopping = false; - private final List onlineAccountsImportQueue = Collections.synchronizedList(new ArrayList<>()); + private final Set onlineAccountsImportQueue = ConcurrentHashMap.newKeySet(); /** * Cache of 'current' online accounts, keyed by timestamp @@ -191,12 +191,9 @@ public class OnlineAccountsManager { LOGGER.debug("Processing online accounts import queue (size: {})", this.onlineAccountsImportQueue.size()); - // Take a copy of onlineAccountsImportQueue so we can safely remove whilst iterating - List onlineAccountsImportQueueCopy = new ArrayList<>(this.onlineAccountsImportQueue); - Set onlineAccountsToAdd = new HashSet<>(); try (final Repository repository = RepositoryManager.getRepository()) { - for (OnlineAccountData onlineAccountData : onlineAccountsImportQueueCopy) { + for (OnlineAccountData onlineAccountData : this.onlineAccountsImportQueue) { if (isStopping) return; @@ -220,19 +217,6 @@ public class OnlineAccountsManager { } } - private boolean importQueueContainsExactMatch(OnlineAccountData acc) { - // Check if an item exists where all properties match exactly - // This is needed because signature and nonce are not compared in OnlineAccountData.equals() - synchronized (onlineAccountsImportQueue) { - return onlineAccountsImportQueue.stream().anyMatch(otherAcc -> - acc.getTimestamp() == otherAcc.getTimestamp() && - Arrays.equals(acc.getPublicKey(), otherAcc.getPublicKey()) && - acc.getNonce() == otherAcc.getNonce() && - Arrays.equals(acc.getSignature(), otherAcc.getSignature()) - ); - } - } - /** * Check if supplied onlineAccountData is superior (i.e. has a nonce value) than existing record. * Two entries are considered equal even if the nonce differs, to prevent multiple variations @@ -855,10 +839,6 @@ public class OnlineAccountsManager { // We have already validated this online account continue; - if (this.importQueueContainsExactMatch(onlineAccountData)) - // Identical online account data already present in queue - continue; - boolean isNewEntry = onlineAccountsImportQueue.add(onlineAccountData); if (isNewEntry) From 765416db71a1b3dd986a108dbfd31713d3004016 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 25 Sep 2022 12:26:00 +0100 Subject: [PATCH 63/83] Yet another attempt to optimize the online accounts import queue processing. The main difference here is that we now remove items from the onlineAccountsImportQueue in a batch, _after_ they have been imported. This prevents duplicates from being added to the queue in the previous time gap between them being removed and imported. --- .../qortal/controller/OnlineAccountsManager.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 47d8cf1b..6fa69a89 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -207,13 +207,20 @@ public class OnlineAccountsManager { boolean isValid = this.isValidCurrentAccount(repository, onlineAccountData); if (isValid) - addAccounts(Arrays.asList(onlineAccountData)); + onlineAccountsToAdd.add(onlineAccountData); - // Remove from queue - onlineAccountsImportQueue.remove(onlineAccountData); + // Don't remove from the queue yet - we'll do this at the end of the process + // This prevents duplicates being added to the queue whilst it's being processed } } catch (DataException e) { LOGGER.error("Repository issue while verifying online accounts", e); + + } finally { + if (!onlineAccountsToAdd.isEmpty()) { + LOGGER.debug("Merging {} validated online accounts from import queue", onlineAccountsToAdd.size()); + addAccounts(onlineAccountsToAdd); + onlineAccountsImportQueue.removeAll(onlineAccountsToAdd); + } } } From 1bb8f1b6d2d68032e02eb05aba6b49e6258f09b9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 25 Sep 2022 12:36:00 +0100 Subject: [PATCH 64/83] Fixed bug in last commit. We need to track items to remove separately from items to add, otherwise invalid accounts remain in the queue. --- .../java/org/qortal/controller/OnlineAccountsManager.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 6fa69a89..686ef514 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -192,6 +192,7 @@ public class OnlineAccountsManager { LOGGER.debug("Processing online accounts import queue (size: {})", this.onlineAccountsImportQueue.size()); Set onlineAccountsToAdd = new HashSet<>(); + Set onlineAccountsToRemove = new HashSet<>(); try (final Repository repository = RepositoryManager.getRepository()) { for (OnlineAccountData onlineAccountData : this.onlineAccountsImportQueue) { if (isStopping) @@ -211,6 +212,7 @@ public class OnlineAccountsManager { // Don't remove from the queue yet - we'll do this at the end of the process // This prevents duplicates being added to the queue whilst it's being processed + onlineAccountsToRemove.add(onlineAccountData); } } catch (DataException e) { LOGGER.error("Repository issue while verifying online accounts", e); @@ -219,7 +221,7 @@ public class OnlineAccountsManager { if (!onlineAccountsToAdd.isEmpty()) { LOGGER.debug("Merging {} validated online accounts from import queue", onlineAccountsToAdd.size()); addAccounts(onlineAccountsToAdd); - onlineAccountsImportQueue.removeAll(onlineAccountsToAdd); + onlineAccountsImportQueue.removeAll(onlineAccountsToRemove); } } } From a9721bab3d735ad376d5296f29833002dcd0e742 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 25 Sep 2022 18:39:56 +0100 Subject: [PATCH 65/83] Fixed issue causing startup of various components to be delayed by 30 seconds. --- .../org/qortal/controller/OnlineAccountsManager.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 686ef514..2644fa66 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -133,17 +133,10 @@ public class OnlineAccountsManager { // Process import queue executor.scheduleWithFixedDelay(this::processOnlineAccountsImportQueue, ONLINE_ACCOUNTS_QUEUE_INTERVAL, ONLINE_ACCOUNTS_QUEUE_INTERVAL, TimeUnit.MILLISECONDS); - // Sleep for some time before scheduling sendOurOnlineAccountsInfo() + // Send our online accounts (using increased initial delay) // This allows some time for initial online account lists to be retrieved, and // reduces the chances of the same nonce being computed twice - try { - Thread.sleep(INITIAL_SLEEP_INTERVAL); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - // Send our online accounts - executor.scheduleAtFixedRate(this::sendOurOnlineAccountsInfo, ONLINE_ACCOUNTS_COMPUTE_INTERVAL, ONLINE_ACCOUNTS_COMPUTE_INTERVAL, TimeUnit.MILLISECONDS); + executor.scheduleAtFixedRate(this::sendOurOnlineAccountsInfo, INITIAL_SLEEP_INTERVAL, ONLINE_ACCOUNTS_COMPUTE_INTERVAL, TimeUnit.MILLISECONDS); } public void shutdown() { From 3890fa849072a5c5d28f265d4235ed32936840d1 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 25 Sep 2022 18:46:33 +0100 Subject: [PATCH 66/83] Renamed constant for consistency --- .../java/org/qortal/controller/OnlineAccountsManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 2644fa66..ff20a8d0 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -53,7 +53,7 @@ public class OnlineAccountsManager { */ private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 3; - private static final long ONLINE_ACCOUNTS_QUEUE_INTERVAL = 100L; //ms + private static final long ONLINE_ACCOUNTS_QUEUE_INTERVAL = 100L; // ms private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms private static final long ONLINE_ACCOUNTS_COMPUTE_INTERVAL = 5 * 1000L; // ms private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 60 * 1000L; // ms @@ -62,7 +62,7 @@ public class OnlineAccountsManager { private static final long ONLINE_ACCOUNTS_BROADCAST_BURST_INTERVAL = 5 * 1000L; // ms private static final long ONLINE_ACCOUNTS_BROADCAST_BURST_LENGTH = 5 * 60 * 1000L; // ms - private static final long INITIAL_SLEEP_INTERVAL = 30 * 1000L; + private static final long ONLINE_ACCOUNTS_COMPUTE_INITIAL_SLEEP_INTERVAL = 30 * 1000L; // ms // MemoryPoW public final int POW_BUFFER_SIZE = 1 * 1024 * 1024; // bytes @@ -136,7 +136,7 @@ public class OnlineAccountsManager { // Send our online accounts (using increased initial delay) // This allows some time for initial online account lists to be retrieved, and // reduces the chances of the same nonce being computed twice - executor.scheduleAtFixedRate(this::sendOurOnlineAccountsInfo, INITIAL_SLEEP_INTERVAL, ONLINE_ACCOUNTS_COMPUTE_INTERVAL, TimeUnit.MILLISECONDS); + executor.scheduleAtFixedRate(this::sendOurOnlineAccountsInfo, ONLINE_ACCOUNTS_COMPUTE_INITIAL_SLEEP_INTERVAL, ONLINE_ACCOUNTS_COMPUTE_INTERVAL, TimeUnit.MILLISECONDS); } public void shutdown() { From 7080b55aacd69138fed21ab8e3370e5088a20b59 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 25 Sep 2022 19:43:56 +0100 Subject: [PATCH 67/83] Reintroduced initial sleep period in block archiver. --- .../java/org/qortal/controller/repository/BlockArchiver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/repository/BlockArchiver.java b/src/main/java/org/qortal/controller/repository/BlockArchiver.java index 8757bf32..63d61ef8 100644 --- a/src/main/java/org/qortal/controller/repository/BlockArchiver.java +++ b/src/main/java/org/qortal/controller/repository/BlockArchiver.java @@ -16,7 +16,7 @@ public class BlockArchiver implements Runnable { private static final Logger LOGGER = LogManager.getLogger(BlockArchiver.class); - private static final long INITIAL_SLEEP_PERIOD = 0L; // TODO: 5 * 60 * 1000L + 1234L; // ms + private static final long INITIAL_SLEEP_PERIOD = 5 * 60 * 1000L + 1234L; // ms public void run() { Thread.currentThread().setName("Block archiver"); From c35c7180d4c15438ca1db2129c03df563fdd24cc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 3 Oct 2022 10:58:47 +0100 Subject: [PATCH 68/83] Return empty levels in GET /addresses/online/levels --- src/main/java/org/qortal/api/resource/AddressesResource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/AddressesResource.java b/src/main/java/org/qortal/api/resource/AddressesResource.java index 4de8d908..468b90a8 100644 --- a/src/main/java/org/qortal/api/resource/AddressesResource.java +++ b/src/main/java/org/qortal/api/resource/AddressesResource.java @@ -205,6 +205,10 @@ public class AddressesResource { try (final Repository repository = RepositoryManager.getRepository()) { List onlineAccountLevels = new ArrayList<>(); + // Prepopulate all levels + for (int i=0; i<=10; i++) + onlineAccountLevels.add(new OnlineAccountLevel(i, 0)); + for (OnlineAccountData onlineAccountData : onlineAccounts) { try { final int minterLevel = Account.getRewardShareEffectiveMintingLevelIncludingLevelZero(repository, onlineAccountData.getPublicKey()); From 1233ba670300c9e6b6b831a034613c5325346120 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 4 Oct 2022 20:08:30 +0100 Subject: [PATCH 69/83] Bump version to 3.6.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e045e0f4..3be7fff3 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.6.0 + 3.6.1 jar true From 10b0f0a0549094a82be158efe88d6a1b18de2fd7 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 5 Oct 2022 15:29:29 +0100 Subject: [PATCH 70/83] Catch JSON exceptions in PirateChainWalletController. This could prevent additional wallets from being initialized if connection was lost while syncing an existing one. --- .../PirateChainWalletController.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/PirateChainWalletController.java b/src/main/java/org/qortal/controller/PirateChainWalletController.java index 1eac4b3a..333c2cda 100644 --- a/src/main/java/org/qortal/controller/PirateChainWalletController.java +++ b/src/main/java/org/qortal/controller/PirateChainWalletController.java @@ -4,6 +4,7 @@ import com.rust.litewalletjni.LiteWalletJni; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.json.JSONException; import org.json.JSONObject; import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.arbitrary.ArbitraryDataReader; @@ -99,14 +100,19 @@ public class PirateChainWalletController extends Thread { LOGGER.debug("Syncing Pirate Chain wallet..."); String response = LiteWalletJni.execute("sync", ""); LOGGER.debug("sync response: {}", response); - JSONObject json = new JSONObject(response); - if (json.has("result")) { - String result = json.getString("result"); - // We may have to set wallet to ready if this is the first ever successful sync - if (Objects.equals(result, "success")) { - this.currentWallet.setReady(true); + try { + JSONObject json = new JSONObject(response); + if (json.has("result")) { + String result = json.getString("result"); + + // We may have to set wallet to ready if this is the first ever successful sync + if (Objects.equals(result, "success")) { + this.currentWallet.setReady(true); + } } + } catch (JSONException e) { + LOGGER.info("Unable to interpret JSON", e); } // Rate limit sync attempts From fdd95eac563beb860797bdb888a6abb98f0cf0c9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 7 Oct 2022 11:05:24 +0100 Subject: [PATCH 71/83] Limit to 240 blocks in syncToPeerChain(). Should fix OutOfMemoryException often seen when syncing from 1000+ blocks behind the chain tip. --- src/main/java/org/qortal/controller/Synchronizer.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index a8d91f52..0fe9a56b 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -1246,7 +1246,14 @@ public class Synchronizer extends Thread { int numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size(); int retryCount = 0; - while (height < peerHeight) { + + // Keep fetching blocks from peer until we reach their tip, or reach a count of MAXIMUM_COMMON_DELTA blocks. + // We need to limit the total number, otherwise too much can be loaded into memory, causing an + // OutOfMemoryException. This is common when syncing from 1000+ blocks behind the chain tip, after starting + // from a small fork that didn't become part of the main chain. This causes the entire sync process to + // use syncToPeerChain(), resulting in potentially thousands of blocks being held in memory if the limit + // below isn't applied. + while (height < peerHeight && peerBlocks.size() <= MAXIMUM_COMMON_DELTA) { if (Controller.isStopping()) return SynchronizationResult.SHUTTING_DOWN; From 8cedf618f45a5d1ab24633e9b32972e663c3dcdd Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 7 Oct 2022 14:46:09 +0100 Subject: [PATCH 72/83] Skip GET_BLOCK_SUMMARIES requests if it can already be fulfilled entirely from the peer's chain tip block summaries cache. Loading from the cache should speed up sync decisions, particularly when choose which peer to sync from. The greater the number of connected peers, the more significant this optimization will be. It should also reduce wasted network requests and data usage. Adding this check prior to making a network request is a simple way to introduce the new cached summaries from BLOCK_SUMMARIES_V2 without having to rewrite a lot of the complex sync / peer comparison logic. Longer term we may want to rewrite that logic to read from the cache directly, but it doesn't make sense to introduce that level of risk at this point time, especially as the Synchronizer may be rewritten soon to prefer longer chains. Even so, this is still quite a high risk commit so lots of testing will be needed. --- .../org/qortal/controller/Synchronizer.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 0fe9a56b..dc70db2a 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -8,6 +8,7 @@ import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -1556,7 +1557,41 @@ public class Synchronizer extends Thread { return SynchronizationResult.OK; } + private List getBlockSummariesFromCache(Peer peer, byte[] parentSignature, int numberRequested) { + List peerSummaries = peer.getChainTipSummaries(); + if (peerSummaries == null) + return null; + + // Check if the requested parent block exists in peer's summaries cache + int parentIndex = IntStream.range(0, peerSummaries.size()).filter(i -> Arrays.equals(peerSummaries.get(i).getSignature(), parentSignature)).findFirst().orElse(-1); + if (parentIndex < 0) + return null; + + // Peer's summaries contains the requested parent, so return summaries after that + // Make sure we have at least one block after the parent block + int summariesAvailable = peerSummaries.size() - parentIndex - 1; + if (summariesAvailable <= 0) + return null; + + // Don't try and return more summaries than we have, or more than were requested + int summariesToReturn = Math.min(numberRequested, summariesAvailable); + int startIndex = parentIndex + 1; + int endIndex = startIndex + summariesToReturn - 1; + if (endIndex > peerSummaries.size() - 1) + return null; + + LOGGER.trace("Serving {} block summaries from cache", summariesToReturn); + return peerSummaries.subList(startIndex, endIndex); + } + private List getBlockSummaries(Peer peer, byte[] parentSignature, int numberRequested) throws InterruptedException { + // We might be able to shortcut the response if we already have the summaries in the peer's chain tip data + List cachedSummaries = this.getBlockSummariesFromCache(peer, parentSignature, numberRequested); + if (cachedSummaries != null && !cachedSummaries.isEmpty()) + return cachedSummaries; + + LOGGER.trace("Requesting {} block summaries from peer {}", numberRequested, peer); + Message getBlockSummariesMessage = new GetBlockSummariesMessage(parentSignature, numberRequested); Message message = peer.getResponse(getBlockSummariesMessage); From 0088ba8485a73c723a9ea4555e0435d42df20a3f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 7 Oct 2022 14:47:46 +0100 Subject: [PATCH 73/83] Reduce INITIAL_BLOCK_STEP from 8 to 7. This allows the first pass to always be served from the peer's cache of 8 summaries. This allows a maximum of 7 to be returned, because the 8th spot is needed for the parent block's signature. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index dc70db2a..ccb3dfdd 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -44,7 +44,7 @@ public class Synchronizer extends Thread { private static final int SYNC_BATCH_SIZE = 1000; // XXX move to Settings? /** Initial jump back of block height when searching for common block with peer */ - private static final int INITIAL_BLOCK_STEP = 8; + private static final int INITIAL_BLOCK_STEP = 7; /** Maximum jump back of block height when searching for common block with peer */ private static final int MAXIMUM_BLOCK_STEP = 128; From 3a18599d8511833872cbd46d3c23e4b3ff81ddf4 Mon Sep 17 00:00:00 2001 From: Nuc1eoN <2538022+Nuc1eoN@users.noreply.github.com> Date: Fri, 7 Oct 2022 23:35:35 +0200 Subject: [PATCH 74/83] Mark start/stop scripts as executables The `start.sh` & `stop.sh` scripts have already been marked as executables in the source folder... But since we have only piped their contents, we need to set correct file permissions again. --- tools/build-zip.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/build-zip.sh b/tools/build-zip.sh index b52b5da7..f423bca1 100755 --- a/tools/build-zip.sh +++ b/tools/build-zip.sh @@ -58,6 +58,9 @@ git show HEAD:log4j2.properties > ${build_dir}/log4j2.properties git show HEAD:start.sh > ${build_dir}/start.sh git show HEAD:stop.sh > ${build_dir}/stop.sh +chmod +x ${build_dir}/start.sh +chmod +x ${build_dir}/stop.sh + printf "{\n}\n" > ${build_dir}/settings.json gtouch -d ${commit_ts%%+??:??} ${build_dir} ${build_dir}/* From 77d60fc33f8171363d58d037044a7bac4ae4152d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 Oct 2022 14:11:28 +0100 Subject: [PATCH 75/83] Revert "Skip GET_BLOCK_SUMMARIES requests if it can already be fulfilled entirely from the peer's chain tip block summaries cache." This reverts commit 8cedf618f45a5d1ab24633e9b32972e663c3dcdd. --- .../org/qortal/controller/Synchronizer.java | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index ccb3dfdd..e4419249 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -8,7 +8,6 @@ import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; -import java.util.stream.IntStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -1557,41 +1556,7 @@ public class Synchronizer extends Thread { return SynchronizationResult.OK; } - private List getBlockSummariesFromCache(Peer peer, byte[] parentSignature, int numberRequested) { - List peerSummaries = peer.getChainTipSummaries(); - if (peerSummaries == null) - return null; - - // Check if the requested parent block exists in peer's summaries cache - int parentIndex = IntStream.range(0, peerSummaries.size()).filter(i -> Arrays.equals(peerSummaries.get(i).getSignature(), parentSignature)).findFirst().orElse(-1); - if (parentIndex < 0) - return null; - - // Peer's summaries contains the requested parent, so return summaries after that - // Make sure we have at least one block after the parent block - int summariesAvailable = peerSummaries.size() - parentIndex - 1; - if (summariesAvailable <= 0) - return null; - - // Don't try and return more summaries than we have, or more than were requested - int summariesToReturn = Math.min(numberRequested, summariesAvailable); - int startIndex = parentIndex + 1; - int endIndex = startIndex + summariesToReturn - 1; - if (endIndex > peerSummaries.size() - 1) - return null; - - LOGGER.trace("Serving {} block summaries from cache", summariesToReturn); - return peerSummaries.subList(startIndex, endIndex); - } - private List getBlockSummaries(Peer peer, byte[] parentSignature, int numberRequested) throws InterruptedException { - // We might be able to shortcut the response if we already have the summaries in the peer's chain tip data - List cachedSummaries = this.getBlockSummariesFromCache(peer, parentSignature, numberRequested); - if (cachedSummaries != null && !cachedSummaries.isEmpty()) - return cachedSummaries; - - LOGGER.trace("Requesting {} block summaries from peer {}", numberRequested, peer); - Message getBlockSummariesMessage = new GetBlockSummariesMessage(parentSignature, numberRequested); Message message = peer.getResponse(getBlockSummariesMessage); From e6bb0b81cff21d2e713185c95b8962d4bb87e50e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 Oct 2022 19:11:20 +0100 Subject: [PATCH 76/83] Revert "Reduce INITIAL_BLOCK_STEP from 8 to 7." This reverts commit 0088ba8485a73c723a9ea4555e0435d42df20a3f. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index e4419249..0fe9a56b 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -43,7 +43,7 @@ public class Synchronizer extends Thread { private static final int SYNC_BATCH_SIZE = 1000; // XXX move to Settings? /** Initial jump back of block height when searching for common block with peer */ - private static final int INITIAL_BLOCK_STEP = 7; + private static final int INITIAL_BLOCK_STEP = 8; /** Maximum jump back of block height when searching for common block with peer */ private static final int MAXIMUM_BLOCK_STEP = 128; From 2d58118d7cfa717a4a6521b9d2fa2bd325c7e5ea Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 Oct 2022 20:11:01 +0100 Subject: [PATCH 77/83] Always use BlockSummariesMessage V1 (instead of V2) when responding to GetBlockSummaries requests. This should hopefully fix a potential issue where peer's chain tip data becomes contaminated with other summary data, causing incorrect sync decisions. --- src/main/java/org/qortal/controller/Controller.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index ce994757..1e028ebc 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1430,9 +1430,7 @@ public class Controller extends Thread { // then we have no blocks after that and can short-circuit with an empty response BlockData chainTip = getChainTip(); if (chainTip != null && Arrays.equals(parentSignature, chainTip.getSignature())) { - Message blockSummariesMessage = peer.getPeersVersion() >= BlockSummariesV2Message.MINIMUM_PEER_VERSION - ? new BlockSummariesV2Message(Collections.emptyList()) - : new BlockSummariesMessage(Collections.emptyList()); + Message blockSummariesMessage = new BlockSummariesMessage(Collections.emptyList()); blockSummariesMessage.setId(message.getId()); @@ -1491,9 +1489,7 @@ public class Controller extends Thread { this.stats.getBlockSummariesStats.fullyFromCache.incrementAndGet(); } - Message blockSummariesMessage = peer.getPeersVersion() >= BlockSummariesV2Message.MINIMUM_PEER_VERSION - ? new BlockSummariesV2Message(blockSummaries) - : new BlockSummariesMessage(blockSummaries); + Message blockSummariesMessage = new BlockSummariesMessage(blockSummaries); blockSummariesMessage.setId(message.getId()); if (!peer.sendMessage(blockSummariesMessage)) peer.disconnect("failed to send block summaries"); From cb1eee8ff5f1f30e647cec69779cbf08dff91f94 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 Oct 2022 20:37:39 +0100 Subject: [PATCH 78/83] GenericUnknownMessage.MINIMUM_PEER_VERSION set to 3.6.1. This should ideally have been set in the 3.6.1 release, but not setting it is unlikely to have caused any problems. --- .../java/org/qortal/network/message/GenericUnknownMessage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/message/GenericUnknownMessage.java b/src/main/java/org/qortal/network/message/GenericUnknownMessage.java index 15faaa1b..dea9f2b8 100644 --- a/src/main/java/org/qortal/network/message/GenericUnknownMessage.java +++ b/src/main/java/org/qortal/network/message/GenericUnknownMessage.java @@ -4,7 +4,7 @@ import java.nio.ByteBuffer; public class GenericUnknownMessage extends Message { - public static final long MINIMUM_PEER_VERSION = 0x03000400cbL; + public static final long MINIMUM_PEER_VERSION = 0x0300060001L; public GenericUnknownMessage() { super(MessageType.GENERIC_UNKNOWN); From 36fcd6792a55352b8d7753dd7d9b8cb16f42d9eb Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 10 Oct 2022 10:28:36 +0100 Subject: [PATCH 79/83] Discard BLOCK_SUMMARIES_V2 messages with an ID (thanks to @catbref for the code) This is a better fix for the "contaminated chain tip summaries" issue. Need to reduce the logging level to debug before release. --- src/main/java/org/qortal/controller/Controller.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 1e028ebc..2146c86b 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1599,6 +1599,17 @@ public class Controller extends Thread { } } + if (message.hasId()) { + /* + * Experimental proof-of-concept: discard messages with ID + * These are 'late' reply messages received after timeout has expired, + * having been passed upwards from Peer to Network to Controller. + * Hence, these are NOT simple "here's my chain tip" broadcasts from other peers. + */ + LOGGER.info("Discarding late {} message with ID {} from {}", message.getType().name(), message.getId(), peer); + return; + } + // Update peer chain tip data peer.setChainTipSummaries(blockSummariesV2Message.getBlockSummaries()); From 10d3176e70694808be0476a95d15804e31fcb948 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 10 Oct 2022 10:28:44 +0100 Subject: [PATCH 80/83] Revert "Always use BlockSummariesMessage V1 (instead of V2) when responding to GetBlockSummaries requests." This reverts commit 2d58118d7cfa717a4a6521b9d2fa2bd325c7e5ea. --- src/main/java/org/qortal/controller/Controller.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 2146c86b..93cbae92 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1430,7 +1430,9 @@ public class Controller extends Thread { // then we have no blocks after that and can short-circuit with an empty response BlockData chainTip = getChainTip(); if (chainTip != null && Arrays.equals(parentSignature, chainTip.getSignature())) { - Message blockSummariesMessage = new BlockSummariesMessage(Collections.emptyList()); + Message blockSummariesMessage = peer.getPeersVersion() >= BlockSummariesV2Message.MINIMUM_PEER_VERSION + ? new BlockSummariesV2Message(Collections.emptyList()) + : new BlockSummariesMessage(Collections.emptyList()); blockSummariesMessage.setId(message.getId()); @@ -1489,7 +1491,9 @@ public class Controller extends Thread { this.stats.getBlockSummariesStats.fullyFromCache.incrementAndGet(); } - Message blockSummariesMessage = new BlockSummariesMessage(blockSummaries); + Message blockSummariesMessage = peer.getPeersVersion() >= BlockSummariesV2Message.MINIMUM_PEER_VERSION + ? new BlockSummariesV2Message(blockSummaries) + : new BlockSummariesMessage(blockSummaries); blockSummariesMessage.setId(message.getId()); if (!peer.sendMessage(blockSummariesMessage)) peer.disconnect("failed to send block summaries"); From d4aaba2293105e63deeb784b87a8dbe566d25724 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 10 Oct 2022 19:06:08 +0100 Subject: [PATCH 81/83] Bump version to 3.6.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3be7fff3..591801e9 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.6.1 + 3.6.2 jar true From 7c15d88cbc23dd45d8c090d286a628c05974af01 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 12 Oct 2022 08:52:58 +0100 Subject: [PATCH 82/83] Fix for issue in BLOCK_SUMMARIES_V2 when sending an empty array of summaries. The BLOCK_SUMMARIES message type would differentiate between an empty response and a missing/invalid response. However, in V2, a response with empty summaries would throw a BufferUnderflowException and be treated by the caller as a null message. This caused problems when trying to find a common block with peers that have diverged by more than 8 blocks. With V1 the caller would know to search back further (e.g. 16 blocks) but in V2 it was treated as "no response" and so the caller would give up instead of increasing the look-back threshold. This fix will identify BLOCK_SUMMARIES_V2 messages with no content, and return an empty array of block summaries instead of a null message. Should be enough to recover any stuck nodes, as long as they haven't diverged more than 240 blocks from the main chain. --- .../qortal/network/message/BlockSummariesV2Message.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/message/BlockSummariesV2Message.java b/src/main/java/org/qortal/network/message/BlockSummariesV2Message.java index 6ed6c8aa..62428cc0 100644 --- a/src/main/java/org/qortal/network/message/BlockSummariesV2Message.java +++ b/src/main/java/org/qortal/network/message/BlockSummariesV2Message.java @@ -68,13 +68,18 @@ public class BlockSummariesV2Message extends Message { } public static Message fromByteBuffer(int id, ByteBuffer bytes) { + List blockSummaries = new ArrayList<>(); + + // If there are no bytes remaining then we can treat this as an empty array of summaries + if (bytes.remaining() == 0) + return new BlockSummariesV2Message(id, blockSummaries); + int height = bytes.getInt(); // Expecting bytes remaining to be exact multiples of BLOCK_SUMMARY_V2_LENGTH if (bytes.remaining() % BLOCK_SUMMARY_V2_LENGTH != 0) throw new BufferUnderflowException(); - List blockSummaries = new ArrayList<>(); while (bytes.hasRemaining()) { byte[] signature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; bytes.get(signature); From 7c7f071eba29240e1b8045df978d1b6fc3f11f60 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 12 Oct 2022 08:54:27 +0100 Subject: [PATCH 83/83] Bump version to 3.6.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 591801e9..5f439cad 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.6.2 + 3.6.3 jar true