Qortal/qortal
1
0
Fork 0
mirror of https://github.com/Qortal/qortal.git synced 2025-03-26 23:44:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

broader exception handling and added verbosity to the logging to 4 more threads

Browse Source
This commit is contained in:
kennycud 2024-10-23 17:39:48 -07:00
parent fa1e86a46b
commit 2629e5f2e8
4 changed files with 189 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java
View File

@ -33,57 +33,59 @@ public class AtStatesTrimmer implements Runnable {
repository.saveChanges(); repository.saveChanges();
while (!Controller.isStopping()) { while (!Controller.isStopping()) {
repository.discardChanges(); try {
repository.discardChanges();
Thread.sleep(Settings.getInstance().getAtStatesTrimInterval()); Thread.sleep(Settings.getInstance().getAtStatesTrimInterval());
BlockData chainTip = Controller.getInstance().getChainTip(); BlockData chainTip = Controller.getInstance().getChainTip();
if (chainTip == null || NTP.getTime() == null) if (chainTip == null || NTP.getTime() == null)
continue; continue;
// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
if (Synchronizer.getInstance().isSynchronizing()) if (Synchronizer.getInstance().isSynchronizing())
continue; continue;
long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime(); long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime();
// We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks // We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks
long chainTrimmableTimestamp = chainTip.getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime(); long chainTrimmableTimestamp = chainTip.getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime();
long upperTrimmableTimestamp = Math.min(currentTrimmableTimestamp, chainTrimmableTimestamp); long upperTrimmableTimestamp = Math.min(currentTrimmableTimestamp, chainTrimmableTimestamp);
int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp); int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp);
int upperBatchHeight = trimStartHeight + Settings.getInstance().getAtStatesTrimBatchSize(); int upperBatchHeight = trimStartHeight + Settings.getInstance().getAtStatesTrimBatchSize();
int upperTrimHeight = Math.min(upperBatchHeight, upperTrimmableHeight); int upperTrimHeight = Math.min(upperBatchHeight, upperTrimmableHeight);
if (trimStartHeight >= upperTrimHeight) if (trimStartHeight >= upperTrimHeight)
continue; continue;
int numAtStatesTrimmed = repository.getATRepository().trimAtStates(trimStartHeight, upperTrimHeight, Settings.getInstance().getAtStatesTrimLimit()); int numAtStatesTrimmed = repository.getATRepository().trimAtStates(trimStartHeight, upperTrimHeight, Settings.getInstance().getAtStatesTrimLimit());
repository.saveChanges(); repository.saveChanges();
if (numAtStatesTrimmed > 0) {
final int finalTrimStartHeight = trimStartHeight;
LOGGER.debug(() -> String.format("Trimmed %d AT state%s between blocks %d and %d",
numAtStatesTrimmed, (numAtStatesTrimmed != 1 ? "s" : ""),
finalTrimStartHeight, upperTrimHeight));
} else {
// Can we move onto next batch?
if (upperTrimmableHeight > upperBatchHeight) {
trimStartHeight = upperBatchHeight;
repository.getATRepository().setAtTrimHeight(trimStartHeight);
maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
repository.saveChanges();
if (numAtStatesTrimmed > 0) {
final int finalTrimStartHeight = trimStartHeight; final int finalTrimStartHeight = trimStartHeight;
LOGGER.debug(() -> String.format("Bumping AT state base trim height to %d", finalTrimStartHeight)); LOGGER.info(() -> String.format("Trimmed %d AT state%s between blocks %d and %d",
numAtStatesTrimmed, (numAtStatesTrimmed != 1 ? "s" : ""),
finalTrimStartHeight, upperTrimHeight));
} else {
// Can we move onto next batch?
if (upperTrimmableHeight > upperBatchHeight) {
trimStartHeight = upperBatchHeight;
repository.getATRepository().setAtTrimHeight(trimStartHeight);
maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
repository.saveChanges();
final int finalTrimStartHeight = trimStartHeight;
LOGGER.info(() -> String.format("Bumping AT state base trim height to %d", finalTrimStartHeight));
}
} }
} catch (Exception e) {
LOGGER.warn("AT States Trimming stopped working. Trying again. Report this error immediately to the developers.", e);
} }
} }
} catch (DataException e) { } catch (Exception e) {
LOGGER.warn(String.format("Repository issue trying to trim AT states: %s", e.getMessage())); LOGGER.error("AT States Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} catch (InterruptedException e) {
// Time to exit
} }
} }

117
src/main/java/org/qortal/controller/repository/BlockArchiver.java
View File

@ -45,71 +45,72 @@ public class BlockArchiver implements Runnable {
LOGGER.info("Starting block archiver from height {}...", startHeight); LOGGER.info("Starting block archiver from height {}...", startHeight);
while (!Controller.isStopping()) { while (!Controller.isStopping()) {
repository.discardChanges();
Thread.sleep(Settings.getInstance().getArchiveInterval());
BlockData chainTip = Controller.getInstance().getChainTip();
if (chainTip == null || NTP.getTime() == null) {
continue;
}
// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
if (Synchronizer.getInstance().isSynchronizing()) {
continue;
}
// Don't attempt to archive if we're not synced yet
final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) {
continue;
}
// Build cache of blocks
try { try {
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); repository.discardChanges();
BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository);
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
switch (result) {
case OK:
// Increment block archive height
startHeight += writer.getWrittenCount();
repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight);
repository.saveChanges();
break;
case STOPPING: Thread.sleep(Settings.getInstance().getArchiveInterval());
return;
// We've reached the limit of the blocks we can archive BlockData chainTip = Controller.getInstance().getChainTip();
// Sleep for a while to allow more to become available if (chainTip == null || NTP.getTime() == null) {
case NOT_ENOUGH_BLOCKS: continue;
// We didn't reach our file size target, so that must mean that we don't have enough blocks
// yet or something went wrong. Sleep for a while and then try again.
repository.discardChanges();
Thread.sleep(60 * 60 * 1000L); // 1 hour
break;
case BLOCK_NOT_FOUND:
// We tried to archive a block that didn't exist. This is a major failure and likely means
// that a bootstrap or re-sync is needed. Try again every minute until then.
LOGGER.info("Error: block not found when building archive. If this error persists, " +
"a bootstrap or re-sync may be needed.");
repository.discardChanges();
Thread.sleep( 60 * 1000L); // 1 minute
break;
} }
} catch (IOException | TransformationException e) { // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
LOGGER.info("Caught exception when creating block cache", e); if (Synchronizer.getInstance().isSynchronizing()) {
} continue;
}
// Don't attempt to archive if we're not synced yet
final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) {
continue;
}
// Build cache of blocks
try {
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository);
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
switch (result) {
case OK:
// Increment block archive height
startHeight += writer.getWrittenCount();
repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight);
repository.saveChanges();
break;
case STOPPING:
return;
// We've reached the limit of the blocks we can archive
// Sleep for a while to allow more to become available
case NOT_ENOUGH_BLOCKS:
// We didn't reach our file size target, so that must mean that we don't have enough blocks
// yet or something went wrong. Sleep for a while and then try again.
repository.discardChanges();
Thread.sleep(60 * 60 * 1000L); // 1 hour
break;
case BLOCK_NOT_FOUND:
// We tried to archive a block that didn't exist. This is a major failure and likely means
// that a bootstrap or re-sync is needed. Try again every minute until then.
LOGGER.info("Error: block not found when building archive. If this error persists, " +
"a bootstrap or re-sync may be needed.");
repository.discardChanges();
Thread.sleep( 60 * 1000L); // 1 minute
break;
}
} catch (IOException | TransformationException e) {
LOGGER.info("Caught exception when creating block cache", e);
}
} catch (Exception e) {
LOGGER.warn("Block Archiving stopped working. Trying again. Report this error immediately to the developers.", e);
}
} }
} catch (DataException e) { } catch (Exception e) {
LOGGER.info("Caught exception when creating block cache", e); LOGGER.error("Block Archiving is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} catch (InterruptedException e) {
// Do nothing
} }
} }

104
src/main/java/org/qortal/controller/repository/BlockPruner.java
View File

@ -48,72 +48,74 @@ public class BlockPruner implements Runnable {
} }
while (!Controller.isStopping()) { while (!Controller.isStopping()) {
repository.discardChanges(); try {
repository.discardChanges();
Thread.sleep(Settings.getInstance().getBlockPruneInterval()); Thread.sleep(Settings.getInstance().getBlockPruneInterval());
BlockData chainTip = Controller.getInstance().getChainTip(); BlockData chainTip = Controller.getInstance().getChainTip();
if (chainTip == null || NTP.getTime() == null) if (chainTip == null || NTP.getTime() == null)
continue; continue;
// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
if (Synchronizer.getInstance().isSynchronizing()) { if (Synchronizer.getInstance().isSynchronizing()) {
continue; continue;
} }
// Don't attempt to prune if we're not synced yet // Don't attempt to prune if we're not synced yet
final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) { if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) {
continue; continue;
} }
// Prune all blocks up until our latest minus pruneBlockLimit // Prune all blocks up until our latest minus pruneBlockLimit
final int ourLatestHeight = chainTip.getHeight(); final int ourLatestHeight = chainTip.getHeight();
int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit();
// In archive mode we are only allowed to trim blocks that have already been archived // In archive mode we are only allowed to trim blocks that have already been archived
if (archiveMode) { if (archiveMode) {
upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1; upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1;
} }
int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize(); int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize();
int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight);
if (pruneStartHeight >= upperPruneHeight) { if (pruneStartHeight >= upperPruneHeight) {
continue; continue;
} }
LOGGER.debug(String.format("Pruning blocks between %d and %d...", pruneStartHeight, upperPruneHeight)); LOGGER.info(String.format("Pruning blocks between %d and %d...", pruneStartHeight, upperPruneHeight));
int numBlocksPruned = repository.getBlockRepository().pruneBlocks(pruneStartHeight, upperPruneHeight); int numBlocksPruned = repository.getBlockRepository().pruneBlocks(pruneStartHeight, upperPruneHeight);
repository.saveChanges();
if (numBlocksPruned > 0) {
LOGGER.debug(String.format("Pruned %d block%s between %d and %d",
numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""),
pruneStartHeight, upperPruneHeight));
} else {
final int nextPruneHeight = upperPruneHeight + 1;
repository.getBlockRepository().setBlockPruneHeight(nextPruneHeight);
repository.saveChanges(); repository.saveChanges();
LOGGER.debug(String.format("Bumping block base prune height to %d", pruneStartHeight));
// Can we move onto next batch? if (numBlocksPruned > 0) {
if (upperPrunableHeight > nextPruneHeight) { LOGGER.info(String.format("Pruned %d block%s between %d and %d",
pruneStartHeight = nextPruneHeight; numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""),
} pruneStartHeight, upperPruneHeight));
else { } else {
// We've pruned up to the upper prunable height final int nextPruneHeight = upperPruneHeight + 1;
// Back off for a while to save CPU for syncing repository.getBlockRepository().setBlockPruneHeight(nextPruneHeight);
repository.discardChanges(); repository.saveChanges();
Thread.sleep(10*60*1000L); LOGGER.info(String.format("Bumping block base prune height to %d", pruneStartHeight));
// Can we move onto next batch?
if (upperPrunableHeight > nextPruneHeight) {
pruneStartHeight = nextPruneHeight;
}
else {
// We've pruned up to the upper prunable height
// Back off for a while to save CPU for syncing
repository.discardChanges();
Thread.sleep(10*60*1000L);
}
} }
} catch (Exception e) {
LOGGER.warn("Block Pruning stopped working. Trying again. Report this error immediately to the developers.", e);
} }
} }
} catch (DataException e) { } catch (Exception e) {
LOGGER.warn(String.format("Repository issue trying to prune blocks: %s", e.getMessage())); LOGGER.error("Block Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} catch (InterruptedException e) {
// Time to exit
} }
} }

72
src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java
View File

@ -33,53 +33,55 @@ public class OnlineAccountsSignaturesTrimmer implements Runnable {
int trimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight(); int trimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight();
while (!Controller.isStopping()) { while (!Controller.isStopping()) {
repository.discardChanges(); try {
repository.discardChanges();
Thread.sleep(Settings.getInstance().getOnlineSignaturesTrimInterval()); Thread.sleep(Settings.getInstance().getOnlineSignaturesTrimInterval());
BlockData chainTip = Controller.getInstance().getChainTip(); BlockData chainTip = Controller.getInstance().getChainTip();
if (chainTip == null || NTP.getTime() == null) if (chainTip == null || NTP.getTime() == null)
continue; continue;
// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
if (Synchronizer.getInstance().isSynchronizing()) if (Synchronizer.getInstance().isSynchronizing())
continue; continue;
// Trim blockchain by removing 'old' online accounts signatures // Trim blockchain by removing 'old' online accounts signatures
long upperTrimmableTimestamp = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime(); long upperTrimmableTimestamp = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime();
int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp); int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp);
int upperBatchHeight = trimStartHeight + Settings.getInstance().getOnlineSignaturesTrimBatchSize(); int upperBatchHeight = trimStartHeight + Settings.getInstance().getOnlineSignaturesTrimBatchSize();
int upperTrimHeight = Math.min(upperBatchHeight, upperTrimmableHeight); int upperTrimHeight = Math.min(upperBatchHeight, upperTrimmableHeight);
if (trimStartHeight >= upperTrimHeight) if (trimStartHeight >= upperTrimHeight)
continue; continue;
int numSigsTrimmed = repository.getBlockRepository().trimOldOnlineAccountsSignatures(trimStartHeight, upperTrimHeight); int numSigsTrimmed = repository.getBlockRepository().trimOldOnlineAccountsSignatures(trimStartHeight, upperTrimHeight);
repository.saveChanges(); repository.saveChanges();
if (numSigsTrimmed > 0) {
final int finalTrimStartHeight = trimStartHeight;
LOGGER.debug(() -> String.format("Trimmed %d online accounts signature%s between blocks %d and %d",
numSigsTrimmed, (numSigsTrimmed != 1 ? "s" : ""),
finalTrimStartHeight, upperTrimHeight));
} else {
// Can we move onto next batch?
if (upperTrimmableHeight > upperBatchHeight) {
trimStartHeight = upperBatchHeight;
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(trimStartHeight);
repository.saveChanges();
if (numSigsTrimmed > 0) {
final int finalTrimStartHeight = trimStartHeight; final int finalTrimStartHeight = trimStartHeight;
LOGGER.debug(() -> String.format("Bumping online accounts signatures base trim height to %d", finalTrimStartHeight)); LOGGER.info(() -> String.format("Trimmed %d online accounts signature%s between blocks %d and %d",
numSigsTrimmed, (numSigsTrimmed != 1 ? "s" : ""),
finalTrimStartHeight, upperTrimHeight));
} else {
// Can we move onto next batch?
if (upperTrimmableHeight > upperBatchHeight) {
trimStartHeight = upperBatchHeight;
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(trimStartHeight);
repository.saveChanges();
final int finalTrimStartHeight = trimStartHeight;
LOGGER.info(() -> String.format("Bumping online accounts signatures base trim height to %d", finalTrimStartHeight));
}
} }
} catch (Exception e) {
LOGGER.warn("Online Accounts Signatures Trimming stopped working. Trying again. Report this error immediately to the developers.", e);
} }
} }
} catch (DataException e) { } catch (Exception e) {
LOGGER.warn(String.format("Repository issue trying to trim online accounts signatures: %s", e.getMessage())); LOGGER.error("Online Accounts Signatures Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
} catch (InterruptedException e) {
// Time to exit
} }
} }