From ba41d84af9eeccbad8c28cca311851b4481dc13c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 26 Jan 2022 22:03:33 +0000 Subject: [PATCH 01/10] Initial attempt to avoid an unnecessary block submission if one of our peers already has a higher weight chain. --- .../org/qortal/controller/BlockMinter.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 616fd611..4ed2a27f 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -20,6 +20,7 @@ import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; +import org.qortal.data.block.CommonBlockData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; import org.qortal.network.Peer; @@ -295,6 +296,20 @@ public class BlockMinter extends Thread { } } + try { + if (this.higherWeightChainExists(repository, bestWeight)) { + LOGGER.info("Higher weight chain found in peers, so not signing a block this round"); + repository.discardChanges(); // Need to remove blockchain lock so that it can actually sync + Thread.sleep(10 * 1000L); // Allow some time for syncing to occur + continue; + } + else { + LOGGER.debug("No higher weight chain found in peers"); + } + } catch (DataException e) { + LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway..."); + } + // Add unconfirmed transactions addUnconfirmedTransactions(repository, newBlock); @@ -462,6 +477,51 @@ public class BlockMinter extends Thread { } } + private BigInteger getOurChainWeightSinceBlock(Repository repository, BlockSummaryData commonBlock, List peerBlockSummaries) throws DataException { + final int commonBlockHeight = commonBlock.getHeight(); + final byte[] commonBlockSig = commonBlock.getSignature(); + int mutualHeight = commonBlockHeight; + + // Fetch our corresponding block summaries + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + List ourBlockSummaries = repository.getBlockRepository() + .getBlockSummaries(commonBlockHeight + 1, ourLatestBlockData.getHeight()); + if (!ourBlockSummaries.isEmpty()) { + Synchronizer.getInstance().populateBlockSummariesMinterLevels(repository, ourBlockSummaries); + } + + if (ourBlockSummaries != null && peerBlockSummaries != null) { + mutualHeight += Math.min(ourBlockSummaries.size(), peerBlockSummaries.size()); + } + return Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight); + } + + private boolean higherWeightChainExists(Repository repository, BigInteger blockCandidateWeight) throws DataException { + if (blockCandidateWeight == null) { + // Can't make decisions without knowing the block candidate weight + return false; + } + List peers = Network.getInstance().getHandshakedPeers(); + // Loop through handshaked peers and check for any new block candidates + for (Peer peer : peers) { + if (peer.getCommonBlockData() != null && peer.getCommonBlockData().getCommonBlockSummary() != null) { + // This peer has common block data + CommonBlockData commonBlockData = peer.getCommonBlockData(); + BlockSummaryData commonBlockSummaryData = commonBlockData.getCommonBlockSummary(); + if (commonBlockData.getChainWeight() != null) { + // The synchronizer has calculated this peer's chain weight + BigInteger ourChainWeightSinceCommonBlock = this.getOurChainWeightSinceBlock(repository, commonBlockSummaryData, commonBlockData.getBlockSummariesAfterCommonBlock()); + BigInteger ourChainWeight = ourChainWeightSinceCommonBlock.add(blockCandidateWeight); + BigInteger peerChainWeight = commonBlockData.getChainWeight(); + if (peerChainWeight.compareTo(ourChainWeight) >= 0) { + // This peer has a higher weight chain than ours (including our block candidate) + return true; + } + } + } + } + return false; + } private static void moderatedLog(Runnable logFunction) { // We only log if logging at TRACE or previous log timeout has expired if (!LOGGER.isTraceEnabled() && lastLogTimestamp != null && lastLogTimestamp + logTimeout > System.currentTimeMillis()) From cbf03d58c875e680c94c02853f6f5706ba6c0dca Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 26 Jan 2022 22:04:09 +0000 Subject: [PATCH 02/10] Made synchronizer method public as it is now also used by the block minter. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index e9090cf0..41ed06a8 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -1516,7 +1516,7 @@ public class Synchronizer extends Thread { return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates()); } - private void populateBlockSummariesMinterLevels(Repository repository, List blockSummaries) throws DataException { + public void populateBlockSummariesMinterLevels(Repository repository, List blockSummaries) throws DataException { final int firstBlockHeight = blockSummaries.get(0).getHeight(); for (int i = 0; i < blockSummaries.size(); ++i) { From 472e1da792d0d14edaf66be53fe523b276cbffe9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 26 Jan 2022 22:34:15 +0000 Subject: [PATCH 03/10] Added debug level logging in higherWeightChainExists() for better visibility. --- .../java/org/qortal/controller/BlockMinter.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 4ed2a27f..f4222750 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -1,6 +1,8 @@ package org.qortal.controller; import java.math.BigInteger; +import java.text.DecimalFormat; +import java.text.NumberFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -501,6 +503,8 @@ public class BlockMinter extends Thread { // Can't make decisions without knowing the block candidate weight return false; } + NumberFormat formatter = new DecimalFormat("0.###E0"); + List peers = Network.getInstance().getHandshakedPeers(); // Loop through handshaked peers and check for any new block candidates for (Peer peer : peers) { @@ -514,10 +518,18 @@ public class BlockMinter extends Thread { BigInteger ourChainWeight = ourChainWeightSinceCommonBlock.add(blockCandidateWeight); BigInteger peerChainWeight = commonBlockData.getChainWeight(); if (peerChainWeight.compareTo(ourChainWeight) >= 0) { - // This peer has a higher weight chain than ours (including our block candidate) + // This peer has a higher weight chain than ours + LOGGER.debug("Peer {} is on a higher weight chain ({}) than ours ({})", peer, formatter.format(peerChainWeight), formatter.format(ourChainWeight)); return true; + + } else { + LOGGER.debug("Peer {} is on a lower weight chain ({}) than ours ({})", peer, formatter.format(peerChainWeight), formatter.format(ourChainWeight)); } + } else { + LOGGER.debug("Peer {} has no chain weight", peer); } + } else { + LOGGER.debug("Peer {} has no common block data", peer); } } return false; From 170244e679ae7c56c201cdbb5c23543fb4b69304 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 17:04:45 +0000 Subject: [PATCH 04/10] More work on higher weight chain detection in the block minter. Added 30 second timeout, so that any errors should self correct. --- .../org/qortal/controller/BlockMinter.java | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index f4222750..ddbf1929 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -78,6 +78,10 @@ public class BlockMinter extends Thread { BlockRepository blockRepository = repository.getBlockRepository(); BlockData previousBlockData = null; + // Vars to keep track of blocks that were skipped due to chain weight + byte[] parentSignatureForLastLowWeightBlock = null; + Long timeOfLastLowWeightBlock = null; + List newBlocks = new ArrayList<>(); // Flags for tracking change in whether minting is possible, @@ -114,6 +118,14 @@ public class BlockMinter extends Thread { if (mintingAccountsData.isEmpty()) continue; + if (parentSignatureForLastLowWeightBlock != null) { + // The last iteration found a higher weight block in the network, so sleep for a while + // to allow is to sync the higher weight chain. We are sleeping here rather than when + // detected as we don't want to hold the blockchain lock open. + LOGGER.info("Sleeping for 10 seconds..."); + Thread.sleep(10 * 1000L); + } + // Disregard minting accounts that are no longer valid, e.g. by transfer/loss of founder flag or account level // Note that minting accounts are actually reward-shares in Qortal Iterator madi = mintingAccountsData.iterator(); @@ -300,10 +312,26 @@ public class BlockMinter extends Thread { try { if (this.higherWeightChainExists(repository, bestWeight)) { - LOGGER.info("Higher weight chain found in peers, so not signing a block this round"); - repository.discardChanges(); // Need to remove blockchain lock so that it can actually sync - Thread.sleep(10 * 1000L); // Allow some time for syncing to occur - continue; + + // Check if the base block has updated since the last time we were here + if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null || + !Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) { + // We've switched to a different chain, so reset the timer + timeOfLastLowWeightBlock = NTP.getTime(); + } + parentSignatureForLastLowWeightBlock = previousBlockData.getSignature(); + + // If less than 30 seconds has passed since first detection the higher weight chain, + // we should skip our block submission to give us the opportunity to sync to the better chain + if (NTP.getTime() - timeOfLastLowWeightBlock < 30*1000L) { + LOGGER.info("Higher weight chain found in peers, so not signing a block this round"); + LOGGER.info("Time since detected: {}", NTP.getTime() - timeOfLastLowWeightBlock); + continue; + } + else { + // More than 30 seconds have passed, so we should submit our block candidate anyway. + LOGGER.info("More than 30 seconds passed, so proceeding to submit block candidate..."); + } } else { LOGGER.debug("No higher weight chain found in peers"); @@ -312,6 +340,11 @@ public class BlockMinter extends Thread { LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway..."); } + // Clear variables that track low weight blocks + parentSignatureForLastLowWeightBlock = null; + timeOfLastLowWeightBlock = null; + + // Add unconfirmed transactions addUnconfirmedTransactions(repository, newBlock); From 60d71863dc5c75f37a38ab0ace41fed75f56fe47 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 17:11:30 +0000 Subject: [PATCH 05/10] Allow 3 seconds for the synchronizer to obtain a blockchain lock, to reduce missed attempts. --- src/main/java/org/qortal/controller/Synchronizer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 41ed06a8..25c10fd6 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -5,6 +5,7 @@ import java.security.SecureRandom; import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @@ -862,7 +863,7 @@ public class Synchronizer extends Thread { // Make sure we're the only thread modifying the blockchain // If we're already synchronizing with another peer then this will also return fast ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - if (!blockchainLock.tryLock()) { + if (!blockchainLock.tryLock(3, TimeUnit.SECONDS)) // Wasn't peer's fault we couldn't sync LOGGER.debug("Synchronizer couldn't acquire blockchain lock"); return SynchronizationResult.NO_BLOCKCHAIN_LOCK; From 483e7549f8c88f1275e09534094304e1eb942c76 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 20:16:52 +0000 Subject: [PATCH 06/10] Revert "Moved log from INFO to DEBUG, as now the synchronizer is on its own thread it can occur more often than before." This reverts commit e2e87766faa0676a6bd334572743f2951918a379. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 25c10fd6..d5c7d1db 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -865,7 +865,7 @@ public class Synchronizer extends Thread { ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); if (!blockchainLock.tryLock(3, TimeUnit.SECONDS)) // Wasn't peer's fault we couldn't sync - LOGGER.debug("Synchronizer couldn't acquire blockchain lock"); + LOGGER.info("Synchronizer couldn't acquire blockchain lock"); return SynchronizationResult.NO_BLOCKCHAIN_LOCK; } From 816b01c1fc6c5902fd835c9bab905a482f1cf2b2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 20:17:46 +0000 Subject: [PATCH 07/10] Fixed issue in rebase --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index d5c7d1db..6a5a5aad 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -863,7 +863,7 @@ public class Synchronizer extends Thread { // Make sure we're the only thread modifying the blockchain // If we're already synchronizing with another peer then this will also return fast ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - if (!blockchainLock.tryLock(3, TimeUnit.SECONDS)) + if (!blockchainLock.tryLock(3, TimeUnit.SECONDS)) { // Wasn't peer's fault we couldn't sync LOGGER.info("Synchronizer couldn't acquire blockchain lock"); return SynchronizationResult.NO_BLOCKCHAIN_LOCK; From a4cbbb38683557f88f3fc4539500140361a814a8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 20:37:26 +0000 Subject: [PATCH 08/10] Moved block minter sleep to later in the process, otherwise it can remain there for longer than expected. --- .../java/org/qortal/controller/BlockMinter.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index ddbf1929..a6aa21a8 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -118,14 +118,6 @@ public class BlockMinter extends Thread { if (mintingAccountsData.isEmpty()) continue; - if (parentSignatureForLastLowWeightBlock != null) { - // The last iteration found a higher weight block in the network, so sleep for a while - // to allow is to sync the higher weight chain. We are sleeping here rather than when - // detected as we don't want to hold the blockchain lock open. - LOGGER.info("Sleeping for 10 seconds..."); - Thread.sleep(10 * 1000L); - } - // Disregard minting accounts that are no longer valid, e.g. by transfer/loss of founder flag or account level // Note that minting accounts are actually reward-shares in Qortal Iterator madi = mintingAccountsData.iterator(); @@ -219,6 +211,14 @@ public class BlockMinter extends Thread { continue; } + if (parentSignatureForLastLowWeightBlock != null) { + // The last iteration found a higher weight block in the network, so sleep for a while + // to allow is to sync the higher weight chain. We are sleeping here rather than when + // detected as we don't want to hold the blockchain lock open. + LOGGER.info("Sleeping for 10 seconds..."); + Thread.sleep(10 * 1000L); + } + for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) { // First block does the AT heavy-lifting if (newBlocks.isEmpty()) { From 55b57021584e817e499a5289902083b64130f1a6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 21:00:27 +0000 Subject: [PATCH 09/10] Invalidate last low weight block signature whenever the previous block data changes. --- src/main/java/org/qortal/controller/BlockMinter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index a6aa21a8..154f7e25 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -195,6 +195,9 @@ public class BlockMinter extends Thread { // Reduce log timeout logTimeout = 10 * 1000L; + + // Last low weight block is no longer valid + parentSignatureForLastLowWeightBlock = null; } // Discard accounts we have already built blocks with From de5f31ac58a28fa706e885bd258e53db96fb5db5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 9 Feb 2022 19:40:20 +0000 Subject: [PATCH 10/10] Don't process file hashes if we're stopping --- .../controller/arbitrary/ArbitraryDataFileRequestThread.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java index 97704ae5..f1015916 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -42,6 +42,10 @@ public class ArbitraryDataFileRequestThread implements Runnable { } private void processFileHashes(Long now) { + if (Controller.isStopping()) { + return; + } + try (final Repository repository = RepositoryManager.getRepository()) { ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance();