diff --git a/src/main/java/org/qortal/controller/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/AtStatesTrimmer.java index b452b3cc..78539813 100644 --- a/src/main/java/org/qortal/controller/AtStatesTrimmer.java +++ b/src/main/java/org/qortal/controller/AtStatesTrimmer.java @@ -2,6 +2,7 @@ package org.qortal.controller; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.pruning.PruneManager; import org.qortal.data.block.BlockData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -22,6 +23,7 @@ public class AtStatesTrimmer implements Runnable { repository.getATRepository().prepareForAtStateTrimming(); repository.saveChanges(); + PruneManager.getInstance().setBuiltLatestATStates(true); while (!Controller.isStopping()) { repository.discardChanges(); diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index bb990b17..2b0a6b8f 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -46,6 +46,7 @@ import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.controller.Synchronizer.SynchronizationResult; +import org.qortal.controller.pruning.PruneManager; import org.qortal.controller.tradebot.TradeBot; import org.qortal.crypto.Crypto; import org.qortal.data.account.MintingAccountData; @@ -357,7 +358,7 @@ public class Controller extends Thread { return this.savedArgs; } - /* package */ static boolean isStopping() { + public static boolean isStopping() { return isStopping; } @@ -1286,6 +1287,13 @@ public class Controller extends Thread { try (final Repository repository = RepositoryManager.getRepository()) { BlockData blockData = repository.getBlockRepository().fromSignature(signature); + if (blockData != null) { + if (PruneManager.getInstance().isBlockPruned(blockData.getHeight(), repository)) { + // If this is a pruned block, we likely only have partial data, so best not to sent it + blockData = null; + } + } + if (blockData == null) { // We don't have this block this.stats.getBlockMessageStats.unknownBlocks.getAndIncrement(); @@ -1407,6 +1415,12 @@ public class Controller extends Thread { BlockData blockData = repository.getBlockRepository().fromReference(parentSignature); + if (PruneManager.getInstance().isBlockPruned(blockData.getHeight(), repository)) { + // If this request contains a pruned block, we likely only have partial data, so best not to sent anything + // We always prune from the oldest first, so it's fine to just check the first block requested + blockData = null; + } + while (blockData != null && blockSummaries.size() < numberRequested) { BlockSummaryData blockSummary = new BlockSummaryData(blockData); blockSummaries.add(blockSummary); diff --git a/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java b/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java new file mode 100644 index 00000000..37f0cd74 --- /dev/null +++ b/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java @@ -0,0 +1,95 @@ +package org.qortal.controller.pruning; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.utils.NTP; + +public class AtStatesPruner implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(AtStatesPruner.class); + + @Override + public void run() { + Thread.currentThread().setName("AT States pruner"); + + if (!Settings.getInstance().isPruningEnabled()) { + return; + } + + try (final Repository repository = RepositoryManager.getRepository()) { + int pruneStartHeight = repository.getATRepository().getAtPruneHeight(); + + // repository.getATRepository().prepareForAtStatePruning(); + // repository.saveChanges(); + + while (!Controller.isStopping()) { + repository.discardChanges(); + + Thread.sleep(Settings.getInstance().getAtStatesPruneInterval()); + + if (PruneManager.getInstance().getBuiltLatestATStates() == false) { + // Wait for latest AT states table to be built first + // This has a dependency on the AtStatesTrimmer running, + // which should be okay, given that it isn't something + // is disabled in normal operation. + continue; + } + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) + continue; + + // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages + if (Controller.getInstance().isSynchronizing()) + continue; + + long currentPrunableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime(); + // We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks + long chainPrunableTimestamp = chainTip.getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime(); + + long upperPrunableTimestamp = Math.min(currentPrunableTimestamp, chainPrunableTimestamp); + int upperPrunableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperPrunableTimestamp); + + int upperBatchHeight = pruneStartHeight + Settings.getInstance().getAtStatesPruneBatchSize(); + int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); + + if (pruneStartHeight >= upperPruneHeight) + continue; + + LOGGER.debug(String.format("Pruning AT states between blocks %d and %d...", pruneStartHeight, upperPruneHeight)); + + int numAtStatesPruned = repository.getATRepository().pruneAtStates(pruneStartHeight, upperPruneHeight); + repository.saveChanges(); + + if (numAtStatesPruned > 0) { + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.debug(() -> String.format("Pruned %d AT state%s between blocks %d and %d", + numAtStatesPruned, (numAtStatesPruned != 1 ? "s" : ""), + finalPruneStartHeight, upperPruneHeight)); + } else { + // Can we move onto next batch? + if (upperPrunableHeight > upperBatchHeight) { + pruneStartHeight = upperBatchHeight; + repository.getATRepository().setAtPruneHeight(pruneStartHeight); + repository.getATRepository().prepareForAtStatePruning(); + repository.saveChanges(); + + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.debug(() -> String.format("Bumping AT state base prune height to %d", finalPruneStartHeight)); + } + } + } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to prune AT states: %s", e.getMessage())); + } catch (InterruptedException e) { + // Time to exit + } + } + +} diff --git a/src/main/java/org/qortal/controller/pruning/PruneManager.java b/src/main/java/org/qortal/controller/pruning/PruneManager.java new file mode 100644 index 00000000..dcd7391d --- /dev/null +++ b/src/main/java/org/qortal/controller/pruning/PruneManager.java @@ -0,0 +1,60 @@ +package org.qortal.controller.pruning; + +import org.qortal.controller.Controller; + +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.settings.Settings; +import org.qortal.utils.DaemonThreadFactory; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class PruneManager { + + private static PruneManager instance; + + private boolean pruningEnabled = Settings.getInstance().isPruningEnabled(); + private int pruneBlockLimit = Settings.getInstance().getPruneBlockLimit(); + private boolean builtLatestATStates = false; + + private PruneManager() { + // Start individual pruning processes + ExecutorService pruneExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory()); + pruneExecutor.execute(new AtStatesPruner()); + } + + public static synchronized PruneManager getInstance() { + if (instance == null) + instance = new PruneManager(); + + return instance; + } + + public boolean isBlockPruned(int height, Repository repository) throws DataException { + if (!this.pruningEnabled) { + return false; + } + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null) { + throw new DataException("Unable to determine chain tip when checking if a block is pruned"); + } + + final int ourLatestHeight = chainTip.getHeight(); + final int latestUnprunedHeight = ourLatestHeight - this.pruneBlockLimit; + + return (height < latestUnprunedHeight); + } + + + public void setBuiltLatestATStates(boolean builtLatestATStates) { + this.builtLatestATStates = builtLatestATStates; + } + + public boolean getBuiltLatestATStates() { + return this.builtLatestATStates; + } + +} diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 558b3aab..6cec0839 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -127,6 +127,23 @@ public interface ATRepository { /** Trims full AT state data between passed heights. Returns number of trimmed rows. */ public int trimAtStates(int minHeight, int maxHeight, int limit) throws DataException; + + /** Returns height of first prunable AT state. */ + public int getAtPruneHeight() throws DataException; + + /** Sets new base height for AT state pruning. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ + public void setAtPruneHeight(int pruneHeight) throws DataException; + + /** Hook to allow repository to prepare/cache info for AT state pruning. */ + public void prepareForAtStatePruning() throws DataException; + + /** Prunes full AT state data between passed heights. Returns number of pruned rows. */ + public int pruneAtStates(int minHeight, int maxHeight) throws DataException; + + /** * Save ATStateData into repository. *

diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index d2461466..d5929311 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -8,6 +8,7 @@ import java.util.Set; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.data.account.AccountData; import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.repository.ATRepository; @@ -682,6 +683,90 @@ public class HSQLDBATRepository implements ATRepository { } } + + @Override + public int getAtPruneHeight() throws DataException { + String sql = "SELECT AT_prune_height FROM DatabaseInfo"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return 0; + + return resultSet.getInt(1); + } catch (SQLException e) { + throw new DataException("Unable to fetch AT state prune height from repository", e); + } + } + + @Override + public void setAtPruneHeight(int pruneHeight) throws DataException { + // trimHeightsLock is to prevent concurrent update on DatabaseInfo + // that could result in "transaction rollback: serialization failure" + synchronized (this.repository.trimHeightsLock) { + String updateSql = "UPDATE DatabaseInfo SET AT_prune_height = ?"; + + try { + this.repository.executeCheckedUpdate(updateSql, pruneHeight); + this.repository.saveChanges(); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to set AT state prune height in repository", e); + } + } + } + + @Override + public void prepareForAtStatePruning() throws DataException { + // Use LatestATStates table that was already built by AtStatesTrimmer + // The AtStatesPruner class checks that this process has completed first + } + + @Override + public int pruneAtStates(int minHeight, int maxHeight) throws DataException { + int deletedCount = 0; + + for (int height=minHeight; height atAddresses = new ArrayList<>(); + String updateSql = "SELECT AT_address FROM LatestATStates WHERE height = ?"; + try (ResultSet resultSet = this.repository.checkedExecute(updateSql, height)) { + if (resultSet != null) { + do { + String atAddress = resultSet.getString(1); + atAddresses.add(atAddress); + + } while (resultSet.next()); + } + } catch (SQLException e) { + throw new DataException("Unable to fetch flagged accounts from repository", e); + } + + List atStates = this.getBlockATStatesAtHeight(height); + for (ATStateData atState : atStates) { + //LOGGER.info("Found atState {} at height {}", atState.getATAddress(), atState.getHeight()); + + if (atAddresses.contains(atState.getATAddress())) { + // We don't want to delete this AT state because it is still active + LOGGER.info("Skipping atState {} at height {}", atState.getATAddress(), atState.getHeight()); + continue; + } + + // Safe to delete everything else for this height + try { + this.repository.delete("ATStates", "AT_address = ? AND height = ?", + atState.getATAddress(), atState.getHeight()); + deletedCount++; + } catch (SQLException e) { + throw new DataException("Unable to delete AT state data from repository", e); + } + } + } + + return deletedCount; + } + + @Override public void save(ATStateData atStateData) throws DataException { // We shouldn't ever save partial ATStateData diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 683a2c3b..94e753e8 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -867,6 +867,10 @@ public class HSQLDBDatabaseUpdates { stmt.execute("CHECKPOINT"); break; } + case 35: + // Support for pruning + stmt.execute("ALTER TABLE DatabaseInfo ADD AT_prune_height INT NOT NULL DEFAULT 0"); + break; default: // nothing to do diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index b8884c6c..f63bdbb9 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -103,6 +103,18 @@ public class Settings { /** Max number of AT states to trim in one go. */ private int atStatesTrimLimit = 4000; // records + /** Whether we should prune old data to reduce database size + * This prevents the node from being able to serve older blocks */ + private boolean pruningEnabled = false; + /** The amount of recent blocks we should keep when pruning */ + private int pruneBlockLimit = 1440; + + /** How often to attempt AT state pruning (ms). */ + private long atStatesPruneInterval = 3219L; // milliseconds + /** Block height range to scan for trimmable AT states.
+ * This has a significant effect on execution time. */ + private int atStatesPruneBatchSize = 10; // blocks + /** How often to attempt online accounts signatures trimming (ms). */ private long onlineSignaturesTrimInterval = 9876L; // milliseconds /** Block height range to scan for trimmable online accounts signatures.
@@ -521,6 +533,22 @@ public class Settings { return this.atStatesTrimLimit; } + public boolean isPruningEnabled() { + return this.pruningEnabled; + } + + public int getPruneBlockLimit() { + return this.pruneBlockLimit; + } + + public long getAtStatesPruneInterval() { + return this.atStatesPruneInterval; + } + + public int getAtStatesPruneBatchSize() { + return this.atStatesPruneBatchSize; + } + public long getOnlineSignaturesTrimInterval() { return this.onlineSignaturesTrimInterval; }