crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

restructuring database connections for better garbage collection, adding in the initial implementation of the balance recorder

Browse Source
This commit is contained in:
kennycud 2024-11-27 17:43:18 -08:00
parent b0d43a1890
commit 84b973773a
12 changed files with 793 additions and 383 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

635
src/main/java/org/qortal/controller/BlockMinter.java
View File

@ -97,364 +97,375 @@ public class BlockMinter extends Thread {
final boolean isSingleNodeTestnet = Settings.getInstance().isSingleNodeTestnet(); final boolean isSingleNodeTestnet = Settings.getInstance().isSingleNodeTestnet();
try (final Repository repository = RepositoryManager.getRepository()) { // Flags for tracking change in whether minting is possible,
// Going to need this a lot... // so we can notify Controller, and further update SysTray, etc.
BlockRepository blockRepository = repository.getBlockRepository(); boolean isMintingPossible = false;
boolean wasMintingPossible = isMintingPossible;
// Flags for tracking change in whether minting is possible, try {
// so we can notify Controller, and further update SysTray, etc.
boolean isMintingPossible = false;
boolean wasMintingPossible = isMintingPossible;
while (running) { while (running) {
if (isMintingPossible != wasMintingPossible) // recreate repository for new loop iteration
Controller.getInstance().onMintingPossibleChange(isMintingPossible); try (final Repository repository = RepositoryManager.getRepository()) {
wasMintingPossible = isMintingPossible; // Going to need this a lot...
BlockRepository blockRepository = repository.getBlockRepository();
try { if (isMintingPossible != wasMintingPossible)
// Free up any repository locks Controller.getInstance().onMintingPossibleChange(isMintingPossible);
repository.discardChanges();
// Sleep for a while. wasMintingPossible = isMintingPossible;
// It's faster on single node testnets, to allow lots of blocks to be minted quickly.
Thread.sleep(isSingleNodeTestnet ? 50 : 1000);
isMintingPossible = false;
final Long now = NTP.getTime();
if (now == null)
continue;
final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
if (minLatestBlockTimestamp == null)
continue;
List<MintingAccountData> mintingAccountsData = repository.getAccountRepository().getMintingAccounts();
// No minting accounts?
if (mintingAccountsData.isEmpty())
continue;
// Disregard minting accounts that are no longer valid, e.g. by transfer/loss of founder flag or account level
// Note that minting accounts are actually reward-shares in Qortal
Iterator<MintingAccountData> madi = mintingAccountsData.iterator();
while (madi.hasNext()) {
MintingAccountData mintingAccountData = madi.next();
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
if (rewardShareData == null) {
// Reward-share doesn't exist - probably cancelled but not yet removed from node's list of minting accounts
madi.remove();
continue;
}
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint(true)) {
// Minting-account component of reward-share can no longer mint - disregard
madi.remove();
continue;
}
// Optional (non-validated) prevention of block submissions below a defined level.
// This is an unvalidated version of Blockchain.minAccountLevelToMint
// and exists only to reduce block candidates by default.
int level = mintingAccount.getEffectiveMintingLevel();
if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) {
madi.remove();
}
}
// Needs a mutable copy of the unmodifiableList
List<Peer> peers = new ArrayList<>(Network.getInstance().getImmutableHandshakedPeers());
BlockData lastBlockData = blockRepository.getLastBlock();
// Disregard peers that have "misbehaved" recently
peers.removeIf(Controller.hasMisbehaved);
// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
if (!Synchronizer.getInstance().getRecoveryMode())
peers.removeIf(Controller.hasNoRecentBlock);
// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
continue;
// If we are stuck on an invalid block, we should allow an alternative to be minted
boolean recoverInvalidBlock = false;
if (Synchronizer.getInstance().timeInvalidBlockLastReceived != null) {
// We've had at least one invalid block
long timeSinceLastValidBlock = NTP.getTime() - Synchronizer.getInstance().timeValidBlockLastReceived;
long timeSinceLastInvalidBlock = NTP.getTime() - Synchronizer.getInstance().timeInvalidBlockLastReceived;
if (timeSinceLastValidBlock > INVALID_BLOCK_RECOVERY_TIMEOUT) {
if (timeSinceLastInvalidBlock < INVALID_BLOCK_RECOVERY_TIMEOUT) {
// Last valid block was more than 10 mins ago, but we've had an invalid block since then
// Assume that the chain has stalled because there is no alternative valid candidate
// Enter recovery mode to allow alternative, valid candidates to be minted
recoverInvalidBlock = true;
}
}
}
// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
if (!Synchronizer.getInstance().getRecoveryMode() && !recoverInvalidBlock)
continue;
// There are enough peers with a recent block and our latest block is recent
// so go ahead and mint a block if possible.
isMintingPossible = true;
// Check blockchain hasn't changed
if (previousBlockData == null || !Arrays.equals(previousBlockData.getSignature(), lastBlockData.getSignature())) {
previousBlockData = lastBlockData;
newBlocks.clear();
// Reduce log timeout
logTimeout = 10 * 1000L;
// Last low weight block is no longer valid
parentSignatureForLastLowWeightBlock = null;
}
// Discard accounts we have already built blocks with
mintingAccountsData.removeIf(mintingAccountData -> newBlocks.stream().anyMatch(newBlock -> Arrays.equals(newBlock.getBlockData().getMinterPublicKey(), mintingAccountData.getPublicKey())));
// Do we need to build any potential new blocks?
List<PrivateKeyAccount> newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList());
// We might need to sit the next block out, if one of our minting accounts signed the previous one
// Skip this check for single node testnets, since they definitely need to mint every block
byte[] previousBlockMinter = previousBlockData.getMinterPublicKey();
boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter));
if (mintedLastBlock && !isSingleNodeTestnet) {
LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one"));
continue;
}
if (parentSignatureForLastLowWeightBlock != null) {
// The last iteration found a higher weight block in the network, so sleep for a while
// to allow is to sync the higher weight chain. We are sleeping here rather than when
// detected as we don't want to hold the blockchain lock open.
LOGGER.info("Sleeping for 10 seconds...");
Thread.sleep(10 * 1000L);
}
for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
// First block does the AT heavy-lifting
if (newBlocks.isEmpty()) {
Block newBlock = Block.mint(repository, previousBlockData, mintingAccount);
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.info("Couldn't build a to-be-minted block"));
continue;
}
newBlocks.add(newBlock);
} else {
// The blocks for other minters require less effort...
Block newBlock = newBlocks.get(0).remint(mintingAccount);
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.error("Couldn't rebuild a to-be-minted block"));
continue;
}
newBlocks.add(newBlock);
}
}
// No potential block candidates?
if (newBlocks.isEmpty())
continue;
// Make sure we're the only thread modifying the blockchain
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
if (!blockchainLock.tryLock(30, TimeUnit.SECONDS)) {
LOGGER.debug("Couldn't acquire blockchain lock even after waiting 30 seconds");
continue;
}
boolean newBlockMinted = false;
Block newBlock = null;
try { try {
// Clear repository session state so we have latest view of data // reset the repository, to the repository recreated for this loop iteration
for( Block newBlock : newBlocks ) newBlock.setRepository(repository);
// Free up any repository locks
repository.discardChanges(); repository.discardChanges();
// Now that we have blockchain lock, do final check that chain hasn't changed // Sleep for a while.
BlockData latestBlockData = blockRepository.getLastBlock(); // It's faster on single node testnets, to allow lots of blocks to be minted quickly.
if (!Arrays.equals(lastBlockData.getSignature(), latestBlockData.getSignature())) Thread.sleep(isSingleNodeTestnet ? 50 : 1000);
isMintingPossible = false;
final Long now = NTP.getTime();
if (now == null)
continue; continue;
List<Block> goodBlocks = new ArrayList<>(); final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
boolean wasInvalidBlockDiscarded = false; if (minLatestBlockTimestamp == null)
Iterator<Block> newBlocksIterator = newBlocks.iterator(); continue;
while (newBlocksIterator.hasNext()) { List<MintingAccountData> mintingAccountsData = repository.getAccountRepository().getMintingAccounts();
Block testBlock = newBlocksIterator.next(); // No minting accounts?
if (mintingAccountsData.isEmpty())
continue;
// Is new block's timestamp valid yet? // Disregard minting accounts that are no longer valid, e.g. by transfer/loss of founder flag or account level
// We do a separate check as some timestamp checks are skipped for testchains // Note that minting accounts are actually reward-shares in Qortal
if (testBlock.isTimestampValid() != ValidationResult.OK) Iterator<MintingAccountData> madi = mintingAccountsData.iterator();
while (madi.hasNext()) {
MintingAccountData mintingAccountData = madi.next();
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
if (rewardShareData == null) {
// Reward-share doesn't exist - probably cancelled but not yet removed from node's list of minting accounts
madi.remove();
continue;
}
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint(true)) {
// Minting-account component of reward-share can no longer mint - disregard
madi.remove();
continue;
}
// Optional (non-validated) prevention of block submissions below a defined level.
// This is an unvalidated version of Blockchain.minAccountLevelToMint
// and exists only to reduce block candidates by default.
int level = mintingAccount.getEffectiveMintingLevel();
if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) {
madi.remove();
}
}
// Needs a mutable copy of the unmodifiableList
List<Peer> peers = new ArrayList<>(Network.getInstance().getImmutableHandshakedPeers());
BlockData lastBlockData = blockRepository.getLastBlock();
// Disregard peers that have "misbehaved" recently
peers.removeIf(Controller.hasMisbehaved);
// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
if (!Synchronizer.getInstance().getRecoveryMode())
peers.removeIf(Controller.hasNoRecentBlock);
// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
continue;
// If we are stuck on an invalid block, we should allow an alternative to be minted
boolean recoverInvalidBlock = false;
if (Synchronizer.getInstance().timeInvalidBlockLastReceived != null) {
// We've had at least one invalid block
long timeSinceLastValidBlock = NTP.getTime() - Synchronizer.getInstance().timeValidBlockLastReceived;
long timeSinceLastInvalidBlock = NTP.getTime() - Synchronizer.getInstance().timeInvalidBlockLastReceived;
if (timeSinceLastValidBlock > INVALID_BLOCK_RECOVERY_TIMEOUT) {
if (timeSinceLastInvalidBlock < INVALID_BLOCK_RECOVERY_TIMEOUT) {
// Last valid block was more than 10 mins ago, but we've had an invalid block since then
// Assume that the chain has stalled because there is no alternative valid candidate
// Enter recovery mode to allow alternative, valid candidates to be minted
recoverInvalidBlock = true;
}
}
}
// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
if (!Synchronizer.getInstance().getRecoveryMode() && !recoverInvalidBlock)
continue; continue;
testBlock.preProcess(); // There are enough peers with a recent block and our latest block is recent
// so go ahead and mint a block if possible.
isMintingPossible = true;
// Is new block valid yet? (Before adding unconfirmed transactions) // Check blockchain hasn't changed
ValidationResult result = testBlock.isValid(); if (previousBlockData == null || !Arrays.equals(previousBlockData.getSignature(), lastBlockData.getSignature())) {
if (result != ValidationResult.OK) { previousBlockData = lastBlockData;
moderatedLog(() -> LOGGER.error(String.format("To-be-minted block invalid '%s' before adding transactions?", result.name()))); newBlocks.clear();
newBlocksIterator.remove(); // Reduce log timeout
wasInvalidBlockDiscarded = true; logTimeout = 10 * 1000L;
/*
* Bail out fast so that we loop around from the top again.
* This gives BlockMinter the possibility to remint this candidate block using another block from newBlocks,
* via the Blocks.remint() method, which avoids having to re-process Block ATs all over again.
* Particularly useful if some aspect of Blocks changes due a timestamp-based feature-trigger (see BlockChain class).
*/
break;
}
goodBlocks.add(testBlock); // Last low weight block is no longer valid
parentSignatureForLastLowWeightBlock = null;
} }
if (wasInvalidBlockDiscarded || goodBlocks.isEmpty()) // Discard accounts we have already built blocks with
mintingAccountsData.removeIf(mintingAccountData -> newBlocks.stream().anyMatch(newBlock -> Arrays.equals(newBlock.getBlockData().getMinterPublicKey(), mintingAccountData.getPublicKey())));
// Do we need to build any potential new blocks?
List<PrivateKeyAccount> newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList());
// We might need to sit the next block out, if one of our minting accounts signed the previous one
// Skip this check for single node testnets, since they definitely need to mint every block
byte[] previousBlockMinter = previousBlockData.getMinterPublicKey();
boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter));
if (mintedLastBlock && !isSingleNodeTestnet) {
LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one"));
continue; continue;
// Pick best block
final int parentHeight = previousBlockData.getHeight();
final byte[] parentBlockSignature = previousBlockData.getSignature();
BigInteger bestWeight = null;
for (int bi = 0; bi < goodBlocks.size(); ++bi) {
BlockData blockData = goodBlocks.get(bi).getBlockData();
BlockSummaryData blockSummaryData = new BlockSummaryData(blockData);
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
blockSummaryData.setMinterLevel(minterLevel);
BigInteger blockWeight = Block.calcBlockWeight(parentHeight, parentBlockSignature, blockSummaryData);
if (bestWeight == null || blockWeight.compareTo(bestWeight) < 0) {
newBlock = goodBlocks.get(bi);
bestWeight = blockWeight;
}
} }
try { if (parentSignatureForLastLowWeightBlock != null) {
if (this.higherWeightChainExists(repository, bestWeight)) { // The last iteration found a higher weight block in the network, so sleep for a while
// to allow is to sync the higher weight chain. We are sleeping here rather than when
// detected as we don't want to hold the blockchain lock open.
LOGGER.info("Sleeping for 10 seconds...");
Thread.sleep(10 * 1000L);
}
// Check if the base block has updated since the last time we were here for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null || // First block does the AT heavy-lifting
!Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) { if (newBlocks.isEmpty()) {
// We've switched to a different chain, so reset the timer Block newBlock = Block.mint(repository, previousBlockData, mintingAccount);
timeOfLastLowWeightBlock = NTP.getTime(); if (newBlock == null) {
} // For some reason we can't mint right now
parentSignatureForLastLowWeightBlock = previousBlockData.getSignature(); moderatedLog(() -> LOGGER.info("Couldn't build a to-be-minted block"));
// If less than 30 seconds has passed since first detection the higher weight chain,
// we should skip our block submission to give us the opportunity to sync to the better chain
if (NTP.getTime() - timeOfLastLowWeightBlock < 30 * 1000L) {
LOGGER.info("Higher weight chain found in peers, so not signing a block this round");
LOGGER.info("Time since detected: {}", NTP.getTime() - timeOfLastLowWeightBlock);
continue; continue;
} else {
// More than 30 seconds have passed, so we should submit our block candidate anyway.
LOGGER.info("More than 30 seconds passed, so proceeding to submit block candidate...");
} }
newBlocks.add(newBlock);
} else { } else {
LOGGER.debug("No higher weight chain found in peers"); // The blocks for other minters require less effort...
Block newBlock = newBlocks.get(0).remint(mintingAccount);
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.error("Couldn't rebuild a to-be-minted block"));
continue;
}
newBlocks.add(newBlock);
} }
} catch (DataException e) {
LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway...");
} }
// Discard any uncommitted changes as a result of the higher weight chain detection // No potential block candidates?
repository.discardChanges(); if (newBlocks.isEmpty())
continue;
// Clear variables that track low weight blocks // Make sure we're the only thread modifying the blockchain
parentSignatureForLastLowWeightBlock = null; ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
timeOfLastLowWeightBlock = null; if (!blockchainLock.tryLock(30, TimeUnit.SECONDS)) {
LOGGER.debug("Couldn't acquire blockchain lock even after waiting 30 seconds");
Long unconfirmedStartTime = NTP.getTime();
// Add unconfirmed transactions
addUnconfirmedTransactions(repository, newBlock);
LOGGER.info(String.format("Adding %d unconfirmed transactions took %d ms", newBlock.getTransactions().size(), (NTP.getTime()-unconfirmedStartTime)));
// Sign to create block's signature
newBlock.sign();
// Is newBlock still valid?
ValidationResult validationResult = newBlock.isValid();
if (validationResult != ValidationResult.OK) {
// No longer valid? Report and discard
LOGGER.error(String.format("To-be-minted block now invalid '%s' after adding unconfirmed transactions?", validationResult.name()));
// Rebuild block candidates, just to be sure
newBlocks.clear();
continue; continue;
} }
// Add to blockchain - something else will notice and broadcast new block to network boolean newBlockMinted = false;
Block newBlock = null;
try { try {
newBlock.process(); // Clear repository session state so we have latest view of data
repository.discardChanges();
repository.saveChanges(); // Now that we have blockchain lock, do final check that chain hasn't changed
BlockData latestBlockData = blockRepository.getLastBlock();
if (!Arrays.equals(lastBlockData.getSignature(), latestBlockData.getSignature()))
continue;
LOGGER.info(String.format("Minted new block: %d", newBlock.getBlockData().getHeight())); List<Block> goodBlocks = new ArrayList<>();
boolean wasInvalidBlockDiscarded = false;
Iterator<Block> newBlocksIterator = newBlocks.iterator();
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey()); while (newBlocksIterator.hasNext()) {
Block testBlock = newBlocksIterator.next();
if (rewardShareData != null) { // Is new block's timestamp valid yet?
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s", // We do a separate check as some timestamp checks are skipped for testchains
newBlock.getBlockData().getHeight(), if (testBlock.isTimestampValid() != ValidationResult.OK)
Base58.encode(newBlock.getBlockData().getSignature()), continue;
Base58.encode(newBlock.getParent().getSignature()),
rewardShareData.getMinter(), testBlock.preProcess();
rewardShareData.getRecipient()));
} else { // Is new block valid yet? (Before adding unconfirmed transactions)
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s", ValidationResult result = testBlock.isValid();
newBlock.getBlockData().getHeight(), if (result != ValidationResult.OK) {
Base58.encode(newBlock.getBlockData().getSignature()), moderatedLog(() -> LOGGER.error(String.format("To-be-minted block invalid '%s' before adding transactions?", result.name())));
Base58.encode(newBlock.getParent().getSignature()),
newBlock.getMinter().getAddress())); newBlocksIterator.remove();
wasInvalidBlockDiscarded = true;
/*
* Bail out fast so that we loop around from the top again.
* This gives BlockMinter the possibility to remint this candidate block using another block from newBlocks,
* via the Blocks.remint() method, which avoids having to re-process Block ATs all over again.
* Particularly useful if some aspect of Blocks changes due a timestamp-based feature-trigger (see BlockChain class).
*/
break;
}
goodBlocks.add(testBlock);
} }
// Notify network after we're released blockchain lock if (wasInvalidBlockDiscarded || goodBlocks.isEmpty())
newBlockMinted = true; continue;
// Notify Controller // Pick best block
repository.discardChanges(); // clear transaction status to prevent deadlocks final int parentHeight = previousBlockData.getHeight();
Controller.getInstance().onNewBlock(newBlock.getBlockData()); final byte[] parentBlockSignature = previousBlockData.getSignature();
} catch (DataException e) {
// Unable to process block - report and discard BigInteger bestWeight = null;
LOGGER.error("Unable to process newly minted block?", e);
newBlocks.clear(); for (int bi = 0; bi < goodBlocks.size(); ++bi) {
} catch (ArithmeticException e) { BlockData blockData = goodBlocks.get(bi).getBlockData();
// Unable to process block - report and discard
LOGGER.error("Unable to process newly minted block?", e); BlockSummaryData blockSummaryData = new BlockSummaryData(blockData);
newBlocks.clear(); int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
blockSummaryData.setMinterLevel(minterLevel);
BigInteger blockWeight = Block.calcBlockWeight(parentHeight, parentBlockSignature, blockSummaryData);
if (bestWeight == null || blockWeight.compareTo(bestWeight) < 0) {
newBlock = goodBlocks.get(bi);
bestWeight = blockWeight;
}
}
try {
if (this.higherWeightChainExists(repository, bestWeight)) {
// Check if the base block has updated since the last time we were here
if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null ||
!Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) {
// We've switched to a different chain, so reset the timer
timeOfLastLowWeightBlock = NTP.getTime();
}
parentSignatureForLastLowWeightBlock = previousBlockData.getSignature();
// If less than 30 seconds has passed since first detection the higher weight chain,
// we should skip our block submission to give us the opportunity to sync to the better chain
if (NTP.getTime() - timeOfLastLowWeightBlock < 30 * 1000L) {
LOGGER.info("Higher weight chain found in peers, so not signing a block this round");
LOGGER.info("Time since detected: {}", NTP.getTime() - timeOfLastLowWeightBlock);
continue;
} else {
// More than 30 seconds have passed, so we should submit our block candidate anyway.
LOGGER.info("More than 30 seconds passed, so proceeding to submit block candidate...");
}
} else {
LOGGER.debug("No higher weight chain found in peers");
}
} catch (DataException e) {
LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway...");
}
// Discard any uncommitted changes as a result of the higher weight chain detection
repository.discardChanges();
// Clear variables that track low weight blocks
parentSignatureForLastLowWeightBlock = null;
timeOfLastLowWeightBlock = null;
Long unconfirmedStartTime = NTP.getTime();
// Add unconfirmed transactions
addUnconfirmedTransactions(repository, newBlock);
LOGGER.info(String.format("Adding %d unconfirmed transactions took %d ms", newBlock.getTransactions().size(), (NTP.getTime() - unconfirmedStartTime)));
// Sign to create block's signature
newBlock.sign();
// Is newBlock still valid?
ValidationResult validationResult = newBlock.isValid();
if (validationResult != ValidationResult.OK) {
// No longer valid? Report and discard
LOGGER.error(String.format("To-be-minted block now invalid '%s' after adding unconfirmed transactions?", validationResult.name()));
// Rebuild block candidates, just to be sure
newBlocks.clear();
continue;
}
// Add to blockchain - something else will notice and broadcast new block to network
try {
newBlock.process();
repository.saveChanges();
LOGGER.info(String.format("Minted new block: %d", newBlock.getBlockData().getHeight()));
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey());
if (rewardShareData != null) {
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s",
newBlock.getBlockData().getHeight(),
Base58.encode(newBlock.getBlockData().getSignature()),
Base58.encode(newBlock.getParent().getSignature()),
rewardShareData.getMinter(),
rewardShareData.getRecipient()));
} else {
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s",
newBlock.getBlockData().getHeight(),
Base58.encode(newBlock.getBlockData().getSignature()),
Base58.encode(newBlock.getParent().getSignature()),
newBlock.getMinter().getAddress()));
}
// Notify network after we're released blockchain lock
newBlockMinted = true;
// Notify Controller
repository.discardChanges(); // clear transaction status to prevent deadlocks
Controller.getInstance().onNewBlock(newBlock.getBlockData());
} catch (DataException e) {
// Unable to process block - report and discard
LOGGER.error("Unable to process newly minted block?", e);
newBlocks.clear();
} catch (ArithmeticException e) {
// Unable to process block - report and discard
LOGGER.error("Unable to process newly minted block?", e);
newBlocks.clear();
}
} finally {
blockchainLock.unlock();
} }
} finally {
blockchainLock.unlock();
}
if (newBlockMinted) { if (newBlockMinted) {
// Broadcast our new chain to network // Broadcast our new chain to network
Network.getInstance().broadcastOurChain(); Network.getInstance().broadcastOurChain();
} }
} catch (InterruptedException e) { } catch (InterruptedException e) {
// We've been interrupted - time to exit // We've been interrupted - time to exit
return; return;
}
} catch (DataException e) {
LOGGER.warn("Repository issue while running block minter - NO LONGER MINTING", e);
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
} }
} }
} catch (DataException e) { } catch (Exception e) {
LOGGER.warn("Repository issue while running block minter - NO LONGER MINTING", e); LOGGER.error(e.getMessage(), e);
} }
} }

37
src/main/java/org/qortal/controller/Controller.java
View File

@ -13,6 +13,7 @@ import org.qortal.block.Block;
import org.qortal.block.BlockChain; import org.qortal.block.BlockChain;
import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.block.BlockChain.BlockTimingByHeight;
import org.qortal.controller.arbitrary.*; import org.qortal.controller.arbitrary.*;
import org.qortal.controller.hsqldb.HSQLDBBalanceRecorder;
import org.qortal.controller.hsqldb.HSQLDBDataCacheManager; import org.qortal.controller.hsqldb.HSQLDBDataCacheManager;
import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
import org.qortal.controller.repository.PruneManager; import org.qortal.controller.repository.PruneManager;
@ -36,7 +37,6 @@ import org.qortal.network.Peer;
import org.qortal.network.PeerAddress; import org.qortal.network.PeerAddress;
import org.qortal.network.message.*; import org.qortal.network.message.*;
import org.qortal.repository.*; import org.qortal.repository.*;
import org.qortal.repository.hsqldb.HSQLDBRepository;
import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory;
import org.qortal.settings.Settings; import org.qortal.settings.Settings;
import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction;
@ -73,6 +73,8 @@ import java.util.stream.Collectors;
public class Controller extends Thread { public class Controller extends Thread {
public static HSQLDBRepositoryFactory REPOSITORY_FACTORY;
static { static {
// This must go before any calls to LogManager/Logger // This must go before any calls to LogManager/Logger
System.setProperty("log4j2.formatMsgNoLookups", "true"); System.setProperty("log4j2.formatMsgNoLookups", "true");
@ -403,23 +405,38 @@ public class Controller extends Thread {
LOGGER.info("Starting repository"); LOGGER.info("Starting repository");
try { try {
RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl()); REPOSITORY_FACTORY = new HSQLDBRepositoryFactory(getRepositoryUrl());
RepositoryManager.setRepositoryFactory(repositoryFactory); RepositoryManager.setRepositoryFactory(REPOSITORY_FACTORY);
RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); RepositoryManager.setRequestedCheckpoint(Boolean.TRUE);
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
// RepositoryManager.rebuildTransactionSequences(repository); // RepositoryManager.rebuildTransactionSequences(repository);
ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, false); ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, false);
}
if( Settings.getInstance().isDbCacheEnabled() ) { if( Settings.getInstance().isDbCacheEnabled() ) {
LOGGER.info("Db Cache Starting ..."); LOGGER.info("Db Cache Starting ...");
HSQLDBDataCacheManager hsqldbDataCacheManager = new HSQLDBDataCacheManager((HSQLDBRepository) repositoryFactory.getRepository()); HSQLDBDataCacheManager hsqldbDataCacheManager = new HSQLDBDataCacheManager();
hsqldbDataCacheManager.start(); hsqldbDataCacheManager.start();
}
else {
LOGGER.info("Db Cache Disabled");
}
if( Settings.getInstance().isBalanceRecorderEnabled() ) {
Optional<HSQLDBBalanceRecorder> recorder = HSQLDBBalanceRecorder.getInstance();
if( recorder.isPresent() ) {
LOGGER.info("Balance Recorder Starting ...");
recorder.get().start();
} }
else { else {
LOGGER.info("Db Cache Disabled"); LOGGER.info("Balance Recorder won't start.");
} }
} }
else {
LOGGER.info("Balance Recorder Disabled");
}
} catch (DataException e) { } catch (DataException e) {
// If exception has no cause or message then repository is in use by some other process. // If exception has no cause or message then repository is in use by some other process.
if (e.getCause() == null && e.getMessage() == null) { if (e.getCause() == null && e.getMessage() == null) {
@ -639,10 +656,8 @@ public class Controller extends Thread {
boolean canBootstrap = Settings.getInstance().getBootstrap(); boolean canBootstrap = Settings.getInstance().getBootstrap();
boolean needsArchiveRebuild = false; boolean needsArchiveRebuild = false;
int checkHeight = 0; int checkHeight = 0;
Repository repository = null;
try { try (final Repository repository = RepositoryManager.getRepository()){
repository = RepositoryManager.getRepository();
needsArchiveRebuild = (repository.getBlockArchiveRepository().fromHeight(2) == null); needsArchiveRebuild = (repository.getBlockArchiveRepository().fromHeight(2) == null);
checkHeight = repository.getBlockRepository().getBlockchainHeight(); checkHeight = repository.getBlockRepository().getBlockchainHeight();
} catch (DataException e) { } catch (DataException e) {

117
src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java Normal file
View File

@ -0,0 +1,117 @@
package org.qortal.controller.hsqldb;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.data.account.AccountBalanceData;
import org.qortal.repository.hsqldb.HSQLDBCacheUtils;
import org.qortal.settings.Settings;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
public class HSQLDBBalanceRecorder extends Thread{
private static final Logger LOGGER = LogManager.getLogger(HSQLDBBalanceRecorder.class);
private static HSQLDBBalanceRecorder SINGLETON = null;
private ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight = new ConcurrentHashMap<>();
private ConcurrentHashMap<String, List<AccountBalanceData>> balancesByAddress = new ConcurrentHashMap<>();
private int priorityRequested;
private int frequency;
private int capacity;
private HSQLDBBalanceRecorder( int priorityRequested, int frequency, int capacity) {
super("Balance Recorder");
this.priorityRequested = priorityRequested;
this.frequency = frequency;
this.capacity = capacity;
}
public static Optional<HSQLDBBalanceRecorder> getInstance() {
if( SINGLETON == null ) {
SINGLETON
= new HSQLDBBalanceRecorder(
Settings.getInstance().getBalanceRecorderPriority(),
Settings.getInstance().getBalanceRecorderFrequency(),
Settings.getInstance().getBalanceRecorderCapacity()
);
}
else if( SINGLETON == null ) {
return Optional.empty();
}
return Optional.of(SINGLETON);
}
@Override
public void run() {
Thread.currentThread().setName("Balance Recorder");
HSQLDBCacheUtils.startRecordingBalances(this.balancesByHeight, this.balancesByAddress, this.priorityRequested, this.frequency, this.capacity);
}
public List<AccountBalanceData> getLatestRecordings(int limit, long offset) {
ArrayList<AccountBalanceData> data;
Optional<Integer> lastHeight = getLastHeight();
if(lastHeight.isPresent() ) {
List<AccountBalanceData> latest = this.balancesByHeight.get(lastHeight.get());
if( latest != null ) {
data = new ArrayList<>(latest.size());
data.addAll(
latest.stream()
.sorted(Comparator.comparingDouble(AccountBalanceData::getBalance).reversed())
.skip(offset)
.limit(limit)
.collect(Collectors.toList())
);
}
else {
data = new ArrayList<>(0);
}
}
else {
data = new ArrayList<>(0);
}
return data;
}
private Optional<Integer> getLastHeight() {
return this.balancesByHeight.keySet().stream().sorted(Comparator.reverseOrder()).findFirst();
}
public List<Integer> getBlocksRecorded() {
return this.balancesByHeight.keySet().stream().collect(Collectors.toList());
}
public List<AccountBalanceData> getAccountBalanceRecordings(String address) {
return this.balancesByAddress.get(address);
}
@Override
public String toString() {
return "HSQLDBBalanceRecorder{" +
"priorityRequested=" + priorityRequested +
", frequency=" + frequency +
", capacity=" + capacity +
'}';
}
}

9
src/main/java/org/qortal/controller/hsqldb/HSQLDBDataCacheManager.java
View File

@ -8,11 +8,7 @@ import org.qortal.settings.Settings;
public class HSQLDBDataCacheManager extends Thread{ public class HSQLDBDataCacheManager extends Thread{
private HSQLDBRepository respository; public HSQLDBDataCacheManager() {}
public HSQLDBDataCacheManager(HSQLDBRepository respository) {
this.respository = respository;
}
@Override @Override
public void run() { public void run() {
@ -20,8 +16,7 @@ public class HSQLDBDataCacheManager extends Thread{
HSQLDBCacheUtils.startCaching( HSQLDBCacheUtils.startCaching(
Settings.getInstance().getDbCacheThreadPriority(), Settings.getInstance().getDbCacheThreadPriority(),
Settings.getInstance().getDbCacheFrequency(), Settings.getInstance().getDbCacheFrequency()
this.respository
); );
} }
} }

30
src/main/java/org/qortal/controller/repository/AtStatesPruner.java
View File

@ -39,15 +39,24 @@ public class AtStatesPruner implements Runnable {
} }
} }
int pruneStartHeight;
int maxLatestAtStatesHeight;
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
int pruneStartHeight = repository.getATRepository().getAtPruneHeight(); pruneStartHeight = repository.getATRepository().getAtPruneHeight();
int maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository); maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
repository.discardChanges(); repository.discardChanges();
repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight); repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
repository.saveChanges(); repository.saveChanges();
} catch (Exception e) {
LOGGER.error("AT States Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
return;
}
while (!Controller.isStopping()) {
try (final Repository repository = RepositoryManager.getRepository()) {
while (!Controller.isStopping()) {
try { try {
repository.discardChanges(); repository.discardChanges();
@ -102,28 +111,25 @@ public class AtStatesPruner implements Runnable {
final int finalPruneStartHeight = pruneStartHeight; final int finalPruneStartHeight = pruneStartHeight;
LOGGER.info(() -> String.format("Bumping AT state base prune height to %d", finalPruneStartHeight)); LOGGER.info(() -> String.format("Bumping AT state base prune height to %d", finalPruneStartHeight));
} } else {
else {
// We've pruned up to the upper prunable height // We've pruned up to the upper prunable height
// Back off for a while to save CPU for syncing // Back off for a while to save CPU for syncing
repository.discardChanges(); repository.discardChanges();
Thread.sleep(5*60*1000L); Thread.sleep(5 * 60 * 1000L);
} }
} }
} catch (InterruptedException e) { } catch (InterruptedException e) {
if(Controller.isStopping()) { if (Controller.isStopping()) {
LOGGER.info("AT States Pruning Shutting Down"); LOGGER.info("AT States Pruning Shutting Down");
} } else {
else {
LOGGER.warn("AT States Pruning interrupted. Trying again. Report this error immediately to the developers.", e); LOGGER.warn("AT States Pruning interrupted. Trying again. Report this error immediately to the developers.", e);
} }
} catch (Exception e) { } catch (Exception e) {
LOGGER.warn("AT States Pruning stopped working. Trying again. Report this error immediately to the developers.", e); LOGGER.warn("AT States Pruning stopped working. Trying again. Report this error immediately to the developers.", e);
} }
} catch(Exception e){
LOGGER.error("AT States Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} }
} catch (Exception e) {
LOGGER.error("AT States Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} }
} }
} }

18
src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java
View File

@ -26,15 +26,23 @@ public class AtStatesTrimmer implements Runnable {
return; return;
} }
int trimStartHeight;
int maxLatestAtStatesHeight;
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
int trimStartHeight = repository.getATRepository().getAtTrimHeight(); trimStartHeight = repository.getATRepository().getAtTrimHeight();
int maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository); maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
repository.discardChanges(); repository.discardChanges();
repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight); repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
repository.saveChanges(); repository.saveChanges();
} catch (Exception e) {
LOGGER.error("AT States Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
return;
}
while (!Controller.isStopping()) { while (!Controller.isStopping()) {
try (final Repository repository = RepositoryManager.getRepository()) {
try { try {
repository.discardChanges(); repository.discardChanges();
@ -92,9 +100,9 @@ public class AtStatesTrimmer implements Runnable {
} catch (Exception e) { } catch (Exception e) {
LOGGER.warn("AT States Trimming stopped working. Trying again. Report this error immediately to the developers.", e); LOGGER.warn("AT States Trimming stopped working. Trying again. Report this error immediately to the developers.", e);
} }
} catch (Exception e) {
LOGGER.error("AT States Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} }
} catch (Exception e) {
LOGGER.error("AT States Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} }
} }

25
src/main/java/org/qortal/controller/repository/BlockArchiver.java
View File

@ -30,11 +30,13 @@ public class BlockArchiver implements Runnable {
return; return;
} }
int startHeight;
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
// Don't even start building until initial rush has ended // Don't even start building until initial rush has ended
Thread.sleep(INITIAL_SLEEP_PERIOD); Thread.sleep(INITIAL_SLEEP_PERIOD);
int startHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); startHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight();
// Don't attempt to archive if we have no ATStatesHeightIndex, as it will be too slow // Don't attempt to archive if we have no ATStatesHeightIndex, as it will be too slow
boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex(); boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex();
@ -43,10 +45,16 @@ public class BlockArchiver implements Runnable {
repository.discardChanges(); repository.discardChanges();
return; return;
} }
} catch (Exception e) {
LOGGER.error("Block Archiving is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
return;
}
LOGGER.info("Starting block archiver from height {}...", startHeight); LOGGER.info("Starting block archiver from height {}...", startHeight);
while (!Controller.isStopping()) {
try (final Repository repository = RepositoryManager.getRepository()) {
while (!Controller.isStopping()) {
try { try {
repository.discardChanges(); repository.discardChanges();
@ -107,20 +115,17 @@ public class BlockArchiver implements Runnable {
LOGGER.info("Caught exception when creating block cache", e); LOGGER.info("Caught exception when creating block cache", e);
} }
} catch (InterruptedException e) { } catch (InterruptedException e) {
if(Controller.isStopping()) { if (Controller.isStopping()) {
LOGGER.info("Block Archiving Shutting Down"); LOGGER.info("Block Archiving Shutting Down");
} } else {
else {
LOGGER.warn("Block Archiving interrupted. Trying again. Report this error immediately to the developers.", e); LOGGER.warn("Block Archiving interrupted. Trying again. Report this error immediately to the developers.", e);
} }
} catch (Exception e) { } catch (Exception e) {
LOGGER.warn("Block Archiving stopped working. Trying again. Report this error immediately to the developers.", e); LOGGER.warn("Block Archiving stopped working. Trying again. Report this error immediately to the developers.", e);
} }
} catch(Exception e){
LOGGER.error("Block Archiving is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} }
} catch (Exception e) {
LOGGER.error("Block Archiving is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} }
} }
} }

18
src/main/java/org/qortal/controller/repository/BlockPruner.java
View File

@ -39,8 +39,10 @@ public class BlockPruner implements Runnable {
} }
} }
int pruneStartHeight;
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
int pruneStartHeight = repository.getBlockRepository().getBlockPruneHeight(); pruneStartHeight = repository.getBlockRepository().getBlockPruneHeight();
// Don't attempt to prune if we have no ATStatesHeightIndex, as it will be too slow // Don't attempt to prune if we have no ATStatesHeightIndex, as it will be too slow
boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex(); boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex();
@ -48,8 +50,15 @@ public class BlockPruner implements Runnable {
LOGGER.info("Unable to start block pruner due to missing ATStatesHeightIndex. Bootstrapping is recommended."); LOGGER.info("Unable to start block pruner due to missing ATStatesHeightIndex. Bootstrapping is recommended.");
return; return;
} }
} catch (Exception e) {
LOGGER.error("Block Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
return;
}
while (!Controller.isStopping()) {
try (final Repository repository = RepositoryManager.getRepository()) {
while (!Controller.isStopping()) {
try { try {
repository.discardChanges(); repository.discardChanges();
@ -122,10 +131,9 @@ public class BlockPruner implements Runnable {
} catch (Exception e) { } catch (Exception e) {
LOGGER.warn("Block Pruning stopped working. Trying again. Report this error immediately to the developers.", e); LOGGER.warn("Block Pruning stopped working. Trying again. Report this error immediately to the developers.", e);
} }
} catch(Exception e){
LOGGER.error("Block Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} }
} catch (Exception e) {
LOGGER.error("Block Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} }
} }
} }

17
src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java
View File

@ -28,13 +28,21 @@ public class OnlineAccountsSignaturesTrimmer implements Runnable {
return; return;
} }
int trimStartHeight;
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
// Don't even start trimming until initial rush has ended // Don't even start trimming until initial rush has ended
Thread.sleep(INITIAL_SLEEP_PERIOD); Thread.sleep(INITIAL_SLEEP_PERIOD);
int trimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight(); trimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight();
} catch (Exception e) {
LOGGER.error("Online Accounts Signatures Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
return;
}
while (!Controller.isStopping()) {
try (final Repository repository = RepositoryManager.getRepository()) {
while (!Controller.isStopping()) {
try { try {
repository.discardChanges(); repository.discardChanges();
@ -88,10 +96,9 @@ public class OnlineAccountsSignaturesTrimmer implements Runnable {
} catch (Exception e) { } catch (Exception e) {
LOGGER.warn("Online Accounts Signatures Trimming stopped working. Trying again. Report this error immediately to the developers.", e); LOGGER.warn("Online Accounts Signatures Trimming stopped working. Trying again. Report this error immediately to the developers.", e);
} }
} catch (Exception e) {
LOGGER.error("Online Accounts Signatures Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} }
} catch (Exception e) {
LOGGER.error("Online Accounts Signatures Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} }
} }
} }

178
src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java
View File

@ -5,10 +5,13 @@ import org.apache.logging.log4j.Logger;
import org.qortal.api.SearchMode; import org.qortal.api.SearchMode;
import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Category;
import org.qortal.arbitrary.misc.Service; import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.data.account.AccountBalanceData;
import org.qortal.data.arbitrary.ArbitraryResourceCache; import org.qortal.data.arbitrary.ArbitraryResourceCache;
import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
import org.qortal.data.arbitrary.ArbitraryResourceStatus; import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.repository.DataException;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
@ -48,6 +51,11 @@ public class HSQLDBCacheUtils {
} }
}; };
private static final String DEFAULT_IDENTIFIER = "default"; private static final String DEFAULT_IDENTIFIER = "default";
private static final int ZERO = 0;
public static final String DB_CACHE_TIMER = "DB Cache Timer";
public static final String DB_CACHE_TIMER_TASK = "DB Cache Timer Task";
public static final String BALANCE_RECORDER_TIMER = "Balance Recorder Timer";
public static final String BALANCE_RECORDER_TIMER_TASK = "Balance Recorder Timer Task";
/** /**
* *
@ -351,13 +359,124 @@ public class HSQLDBCacheUtils {
* Start Caching * Start Caching
* *
* @param priorityRequested the thread priority to fill cache in * @param priorityRequested the thread priority to fill cache in
* @param frequency the frequency to fill the cache (in seconds) * @param frequency the frequency to fill the cache (in seconds)
* @param respository the data source
* *
* @return the data cache * @return the data cache
*/ */
public static void startCaching(int priorityRequested, int frequency, HSQLDBRepository respository) { public static void startCaching(int priorityRequested, int frequency) {
Timer timer = buildTimer(DB_CACHE_TIMER, priorityRequested);
TimerTask task = new TimerTask() {
@Override
public void run() {
Thread.currentThread().setName(DB_CACHE_TIMER_TASK);
try (final HSQLDBRepository respository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) {
fillCache(ArbitraryResourceCache.getInstance(), respository);
}
catch( DataException e ) {
LOGGER.error(e.getMessage(), e);
}
}
};
// delay 1 second
timer.scheduleAtFixedRate(task, 1000, frequency * 1000);
}
/**
* Start Recording Balances
*
* @param queue the queue to add to, remove oldest data if necssary
* @param repository the db repsoitory
* @param priorityRequested the requested thread priority
* @param frequency the recording frequencies, in minutes
*/
public static void startRecordingBalances(
final ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight,
final ConcurrentHashMap<String, List<AccountBalanceData>> balancesByAddress,
int priorityRequested,
int frequency,
int capacity) {
Timer timer = buildTimer(BALANCE_RECORDER_TIMER, priorityRequested);
TimerTask task = new TimerTask() {
@Override
public void run() {
Thread.currentThread().setName(BALANCE_RECORDER_TIMER_TASK);
try (final HSQLDBRepository repository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) {
while (balancesByHeight.size() > capacity + 1) {
Optional<Integer> firstHeight = balancesByHeight.keySet().stream().sorted().findFirst();
if (firstHeight.isPresent()) balancesByHeight.remove(firstHeight.get());
}
// get current balances
List<AccountBalanceData> accountBalances = getAccountBalances(repository);
// get anyone of the balances
Optional<AccountBalanceData> data = accountBalances.stream().findAny();
// if there are any balances, then record them
if (data.isPresent()) {
// map all new balances to the current height
balancesByHeight.put(data.get().getHeight(), accountBalances);
// for each new balance, map to address
for (AccountBalanceData accountBalance : accountBalances) {
// get recorded balances for this address
List<AccountBalanceData> establishedBalances
= balancesByAddress.getOrDefault(accountBalance.getAddress(), new ArrayList<>(0));
// start a new list of recordings for this address, add the new balance and add the established
// balances
List<AccountBalanceData> balances = new ArrayList<>(establishedBalances.size() + 1);
balances.add(accountBalance);
balances.addAll(establishedBalances);
// reset tha balances for this address
balancesByAddress.put(accountBalance.getAddress(), balances);
// TODO: reduce account balances to capacity
}
// reduce height balances to capacity
while( balancesByHeight.size() > capacity ) {
Optional<Integer> lowestHeight
= balancesByHeight.entrySet().stream()
.min(Comparator.comparingInt(Map.Entry::getKey))
.map(Map.Entry::getKey);
if (lowestHeight.isPresent()) balancesByHeight.entrySet().remove(lowestHeight);
}
}
} catch (DataException e) {
LOGGER.error(e.getMessage(), e);
}
}
};
// wait 5 minutes
timer.scheduleAtFixedRate(task, 300_000, frequency * 60_000);
}
/**
* Build Timer
*
* Build a timer for scheduling a timer task.
*
* @param name the name for the thread running the timer task
* @param priorityRequested the priority for the thread running the timer task
*
* @return a timer for scheduling a timer task
*/
private static Timer buildTimer( final String name, int priorityRequested) {
// ensure priority is in between 1-10 // ensure priority is in between 1-10
final int priority = Math.max(0, Math.min(10, priorityRequested)); final int priority = Math.max(0, Math.min(10, priorityRequested));
@ -365,7 +484,7 @@ public class HSQLDBCacheUtils {
Timer timer = new Timer(true) { // 'true' to make the Timer daemon Timer timer = new Timer(true) { // 'true' to make the Timer daemon
@Override @Override
public void schedule(TimerTask task, long delay) { public void schedule(TimerTask task, long delay) {
Thread thread = new Thread(task) { Thread thread = new Thread(task, name) {
@Override @Override
public void run() { public void run() {
this.setPriority(priority); this.setPriority(priority);
@ -376,17 +495,7 @@ public class HSQLDBCacheUtils {
thread.start(); thread.start();
} }
}; };
return timer;
TimerTask task = new TimerTask() {
@Override
public void run() {
fillCache(ArbitraryResourceCache.getInstance(), respository);
}
};
// delay 1 second
timer.scheduleAtFixedRate(task, 1000, frequency * 1000);
} }
/** /**
@ -541,4 +650,43 @@ public class HSQLDBCacheUtils {
return resources; return resources;
} }
public static List<AccountBalanceData> getAccountBalances(HSQLDBRepository repository) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT account, balance, height ");
sql.append("FROM ACCOUNTBALANCES as balances ");
sql.append("JOIN (SELECT height FROM BLOCKS ORDER BY height DESC LIMIT 1) AS max_height ON true ");
sql.append("WHERE asset_id=0");
List<AccountBalanceData> data = new ArrayList<>();
LOGGER.info( "Getting account balances ...");
try {
Statement statement = repository.connection.createStatement();
ResultSet resultSet = statement.executeQuery(sql.toString());
if (resultSet == null || !resultSet.next())
return new ArrayList<>(0);
do {
String account = resultSet.getString(1);
long balance = resultSet.getLong(2);
int height = resultSet.getInt(3);
data.add(new AccountBalanceData(account, ZERO, balance, height));
} while (resultSet.next());
} catch (SQLException e) {
LOGGER.warn(e.getMessage());
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
LOGGER.info("Retrieved account balances: count = " + data.size());
return data;
}
} }

66
src/main/java/org/qortal/repository/hsqldb/HSQLDBUtils.java Normal file
View File

@ -0,0 +1,66 @@
package org.qortal.repository.hsqldb;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.model.DatasetStatus;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
/**
* Class HSQLDBUtils
*/
public class HSQLDBUtils {
private static final Logger LOGGER = LogManager.getLogger(HSQLDBUtils.class);
/**
* Get Row Counts
*
* Get row counts for each table in the database
*
* @param connection the database
*
* @return the table name and current row count paired together for each table
*/
public static List<DatasetStatus> getRowCounts(Connection connection) {
List<DatasetStatus> dbTables = new ArrayList<>();
try {
// Get the database metadata
DatabaseMetaData metaData = connection.getMetaData();
// Retrieve a list of all tables in the database
ResultSet tables = metaData.getTables(null, null, "%", null);
// Process each table and get its row count
while (tables.next()) {
String tableName = tables.getString(3);
// skip system tables
String tableType = tables.getString("TABLE_TYPE");
if (tableType.equals("SYSTEM TABLE")) continue;
// Execute a query to count the rows in the table
Statement statement = connection.createStatement();
ResultSet resultSet = statement.executeQuery("SELECT COUNT(*) FROM " + tableName);
// Get the row count from the ResultSet
if (resultSet.next()) dbTables.add(new DatasetStatus(tableName, resultSet.getLong(1)));
// Close the statement
statement.close();
}
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
return dbTables;
}
}

26
src/main/java/org/qortal/settings/Settings.java
View File

@ -213,7 +213,7 @@ public class Settings {
public long recoveryModeTimeout = 9999999999999L; public long recoveryModeTimeout = 9999999999999L;
/** Minimum peer version number required in order to sync with them */ /** Minimum peer version number required in order to sync with them */
private String minPeerVersion = "4.6.2"; private String minPeerVersion = "4.6.3";
/** Whether to allow connections with peers below minPeerVersion /** Whether to allow connections with peers below minPeerVersion
* If true, we won't sync with them but they can still sync with us, and will show in the peers list * If true, we won't sync with them but they can still sync with us, and will show in the peers list
* If false, sync will be blocked both ways, and they will not appear in the peers list */ * If false, sync will be blocked both ways, and they will not appear in the peers list */
@ -441,6 +441,14 @@ public class Settings {
*/ */
private long archivingPause = 3000; private long archivingPause = 3000;
private boolean balanceRecorderEnabled = false;
private int balanceRecorderPriority = 1;
private int balanceRecorderFrequency = 2*60*1000;
private int balanceRecorderCapacity = 1000;
// Domain mapping // Domain mapping
public static class ThreadLimit { public static class ThreadLimit {
private String messageType; private String messageType;
@ -1230,4 +1238,20 @@ public class Settings {
public long getArchivingPause() { public long getArchivingPause() {
return archivingPause; return archivingPause;
} }
public int getBalanceRecorderPriority() {
return balanceRecorderPriority;
}
public int getBalanceRecorderFrequency() {
return balanceRecorderFrequency;
}
public int getBalanceRecorderCapacity() {
return balanceRecorderCapacity;
}
public boolean isBalanceRecorderEnabled() {
return balanceRecorderEnabled;
}
} }