From e7e4cb75796ee9c5a3450fffd6a4984a501d062e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 22 Aug 2021 08:01:02 +0100 Subject: [PATCH 01/22] Started work on pruning mode (top-only-sync) Initially just deleting old and unused AT states, to get this table under control. I have had to delete them individually as the table can't handle complex queries due to its size. Nodes in pruning mode will be unable to serve older blocks to peers. --- .../qortal/controller/AtStatesTrimmer.java | 2 + .../org/qortal/controller/Controller.java | 16 +++- .../controller/pruning/AtStatesPruner.java | 95 +++++++++++++++++++ .../controller/pruning/PruneManager.java | 60 ++++++++++++ .../org/qortal/repository/ATRepository.java | 17 ++++ .../repository/hsqldb/HSQLDBATRepository.java | 85 +++++++++++++++++ .../hsqldb/HSQLDBDatabaseUpdates.java | 4 + .../java/org/qortal/settings/Settings.java | 28 ++++++ 8 files changed, 306 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/qortal/controller/pruning/AtStatesPruner.java create mode 100644 src/main/java/org/qortal/controller/pruning/PruneManager.java diff --git a/src/main/java/org/qortal/controller/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/AtStatesTrimmer.java index b452b3cc..78539813 100644 --- a/src/main/java/org/qortal/controller/AtStatesTrimmer.java +++ b/src/main/java/org/qortal/controller/AtStatesTrimmer.java @@ -2,6 +2,7 @@ package org.qortal.controller; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.pruning.PruneManager; import org.qortal.data.block.BlockData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -22,6 +23,7 @@ public class AtStatesTrimmer implements Runnable { repository.getATRepository().prepareForAtStateTrimming(); repository.saveChanges(); + PruneManager.getInstance().setBuiltLatestATStates(true); while (!Controller.isStopping()) { repository.discardChanges(); diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index bb990b17..2b0a6b8f 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -46,6 +46,7 @@ import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.controller.Synchronizer.SynchronizationResult; +import org.qortal.controller.pruning.PruneManager; import org.qortal.controller.tradebot.TradeBot; import org.qortal.crypto.Crypto; import org.qortal.data.account.MintingAccountData; @@ -357,7 +358,7 @@ public class Controller extends Thread { return this.savedArgs; } - /* package */ static boolean isStopping() { + public static boolean isStopping() { return isStopping; } @@ -1286,6 +1287,13 @@ public class Controller extends Thread { try (final Repository repository = RepositoryManager.getRepository()) { BlockData blockData = repository.getBlockRepository().fromSignature(signature); + if (blockData != null) { + if (PruneManager.getInstance().isBlockPruned(blockData.getHeight(), repository)) { + // If this is a pruned block, we likely only have partial data, so best not to sent it + blockData = null; + } + } + if (blockData == null) { // We don't have this block this.stats.getBlockMessageStats.unknownBlocks.getAndIncrement(); @@ -1407,6 +1415,12 @@ public class Controller extends Thread { BlockData blockData = repository.getBlockRepository().fromReference(parentSignature); + if (PruneManager.getInstance().isBlockPruned(blockData.getHeight(), repository)) { + // If this request contains a pruned block, we likely only have partial data, so best not to sent anything + // We always prune from the oldest first, so it's fine to just check the first block requested + blockData = null; + } + while (blockData != null && blockSummaries.size() < numberRequested) { BlockSummaryData blockSummary = new BlockSummaryData(blockData); blockSummaries.add(blockSummary); diff --git a/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java b/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java new file mode 100644 index 00000000..37f0cd74 --- /dev/null +++ b/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java @@ -0,0 +1,95 @@ +package org.qortal.controller.pruning; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.utils.NTP; + +public class AtStatesPruner implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(AtStatesPruner.class); + + @Override + public void run() { + Thread.currentThread().setName("AT States pruner"); + + if (!Settings.getInstance().isPruningEnabled()) { + return; + } + + try (final Repository repository = RepositoryManager.getRepository()) { + int pruneStartHeight = repository.getATRepository().getAtPruneHeight(); + + // repository.getATRepository().prepareForAtStatePruning(); + // repository.saveChanges(); + + while (!Controller.isStopping()) { + repository.discardChanges(); + + Thread.sleep(Settings.getInstance().getAtStatesPruneInterval()); + + if (PruneManager.getInstance().getBuiltLatestATStates() == false) { + // Wait for latest AT states table to be built first + // This has a dependency on the AtStatesTrimmer running, + // which should be okay, given that it isn't something + // is disabled in normal operation. + continue; + } + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) + continue; + + // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages + if (Controller.getInstance().isSynchronizing()) + continue; + + long currentPrunableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime(); + // We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks + long chainPrunableTimestamp = chainTip.getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime(); + + long upperPrunableTimestamp = Math.min(currentPrunableTimestamp, chainPrunableTimestamp); + int upperPrunableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperPrunableTimestamp); + + int upperBatchHeight = pruneStartHeight + Settings.getInstance().getAtStatesPruneBatchSize(); + int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); + + if (pruneStartHeight >= upperPruneHeight) + continue; + + LOGGER.debug(String.format("Pruning AT states between blocks %d and %d...", pruneStartHeight, upperPruneHeight)); + + int numAtStatesPruned = repository.getATRepository().pruneAtStates(pruneStartHeight, upperPruneHeight); + repository.saveChanges(); + + if (numAtStatesPruned > 0) { + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.debug(() -> String.format("Pruned %d AT state%s between blocks %d and %d", + numAtStatesPruned, (numAtStatesPruned != 1 ? "s" : ""), + finalPruneStartHeight, upperPruneHeight)); + } else { + // Can we move onto next batch? + if (upperPrunableHeight > upperBatchHeight) { + pruneStartHeight = upperBatchHeight; + repository.getATRepository().setAtPruneHeight(pruneStartHeight); + repository.getATRepository().prepareForAtStatePruning(); + repository.saveChanges(); + + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.debug(() -> String.format("Bumping AT state base prune height to %d", finalPruneStartHeight)); + } + } + } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to prune AT states: %s", e.getMessage())); + } catch (InterruptedException e) { + // Time to exit + } + } + +} diff --git a/src/main/java/org/qortal/controller/pruning/PruneManager.java b/src/main/java/org/qortal/controller/pruning/PruneManager.java new file mode 100644 index 00000000..dcd7391d --- /dev/null +++ b/src/main/java/org/qortal/controller/pruning/PruneManager.java @@ -0,0 +1,60 @@ +package org.qortal.controller.pruning; + +import org.qortal.controller.Controller; + +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.settings.Settings; +import org.qortal.utils.DaemonThreadFactory; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class PruneManager { + + private static PruneManager instance; + + private boolean pruningEnabled = Settings.getInstance().isPruningEnabled(); + private int pruneBlockLimit = Settings.getInstance().getPruneBlockLimit(); + private boolean builtLatestATStates = false; + + private PruneManager() { + // Start individual pruning processes + ExecutorService pruneExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory()); + pruneExecutor.execute(new AtStatesPruner()); + } + + public static synchronized PruneManager getInstance() { + if (instance == null) + instance = new PruneManager(); + + return instance; + } + + public boolean isBlockPruned(int height, Repository repository) throws DataException { + if (!this.pruningEnabled) { + return false; + } + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null) { + throw new DataException("Unable to determine chain tip when checking if a block is pruned"); + } + + final int ourLatestHeight = chainTip.getHeight(); + final int latestUnprunedHeight = ourLatestHeight - this.pruneBlockLimit; + + return (height < latestUnprunedHeight); + } + + + public void setBuiltLatestATStates(boolean builtLatestATStates) { + this.builtLatestATStates = builtLatestATStates; + } + + public boolean getBuiltLatestATStates() { + return this.builtLatestATStates; + } + +} diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 558b3aab..6cec0839 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -127,6 +127,23 @@ public interface ATRepository { /** Trims full AT state data between passed heights. Returns number of trimmed rows. */ public int trimAtStates(int minHeight, int maxHeight, int limit) throws DataException; + + /** Returns height of first prunable AT state. */ + public int getAtPruneHeight() throws DataException; + + /** Sets new base height for AT state pruning. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ + public void setAtPruneHeight(int pruneHeight) throws DataException; + + /** Hook to allow repository to prepare/cache info for AT state pruning. */ + public void prepareForAtStatePruning() throws DataException; + + /** Prunes full AT state data between passed heights. Returns number of pruned rows. */ + public int pruneAtStates(int minHeight, int maxHeight) throws DataException; + + /** * Save ATStateData into repository. *

diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index d2461466..d5929311 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -8,6 +8,7 @@ import java.util.Set; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.data.account.AccountData; import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.repository.ATRepository; @@ -682,6 +683,90 @@ public class HSQLDBATRepository implements ATRepository { } } + + @Override + public int getAtPruneHeight() throws DataException { + String sql = "SELECT AT_prune_height FROM DatabaseInfo"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return 0; + + return resultSet.getInt(1); + } catch (SQLException e) { + throw new DataException("Unable to fetch AT state prune height from repository", e); + } + } + + @Override + public void setAtPruneHeight(int pruneHeight) throws DataException { + // trimHeightsLock is to prevent concurrent update on DatabaseInfo + // that could result in "transaction rollback: serialization failure" + synchronized (this.repository.trimHeightsLock) { + String updateSql = "UPDATE DatabaseInfo SET AT_prune_height = ?"; + + try { + this.repository.executeCheckedUpdate(updateSql, pruneHeight); + this.repository.saveChanges(); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to set AT state prune height in repository", e); + } + } + } + + @Override + public void prepareForAtStatePruning() throws DataException { + // Use LatestATStates table that was already built by AtStatesTrimmer + // The AtStatesPruner class checks that this process has completed first + } + + @Override + public int pruneAtStates(int minHeight, int maxHeight) throws DataException { + int deletedCount = 0; + + for (int height=minHeight; height atAddresses = new ArrayList<>(); + String updateSql = "SELECT AT_address FROM LatestATStates WHERE height = ?"; + try (ResultSet resultSet = this.repository.checkedExecute(updateSql, height)) { + if (resultSet != null) { + do { + String atAddress = resultSet.getString(1); + atAddresses.add(atAddress); + + } while (resultSet.next()); + } + } catch (SQLException e) { + throw new DataException("Unable to fetch flagged accounts from repository", e); + } + + List atStates = this.getBlockATStatesAtHeight(height); + for (ATStateData atState : atStates) { + //LOGGER.info("Found atState {} at height {}", atState.getATAddress(), atState.getHeight()); + + if (atAddresses.contains(atState.getATAddress())) { + // We don't want to delete this AT state because it is still active + LOGGER.info("Skipping atState {} at height {}", atState.getATAddress(), atState.getHeight()); + continue; + } + + // Safe to delete everything else for this height + try { + this.repository.delete("ATStates", "AT_address = ? AND height = ?", + atState.getATAddress(), atState.getHeight()); + deletedCount++; + } catch (SQLException e) { + throw new DataException("Unable to delete AT state data from repository", e); + } + } + } + + return deletedCount; + } + + @Override public void save(ATStateData atStateData) throws DataException { // We shouldn't ever save partial ATStateData diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 683a2c3b..94e753e8 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -867,6 +867,10 @@ public class HSQLDBDatabaseUpdates { stmt.execute("CHECKPOINT"); break; } + case 35: + // Support for pruning + stmt.execute("ALTER TABLE DatabaseInfo ADD AT_prune_height INT NOT NULL DEFAULT 0"); + break; default: // nothing to do diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index b8884c6c..f63bdbb9 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -103,6 +103,18 @@ public class Settings { /** Max number of AT states to trim in one go. */ private int atStatesTrimLimit = 4000; // records + /** Whether we should prune old data to reduce database size + * This prevents the node from being able to serve older blocks */ + private boolean pruningEnabled = false; + /** The amount of recent blocks we should keep when pruning */ + private int pruneBlockLimit = 1440; + + /** How often to attempt AT state pruning (ms). */ + private long atStatesPruneInterval = 3219L; // milliseconds + /** Block height range to scan for trimmable AT states.
+ * This has a significant effect on execution time. */ + private int atStatesPruneBatchSize = 10; // blocks + /** How often to attempt online accounts signatures trimming (ms). */ private long onlineSignaturesTrimInterval = 9876L; // milliseconds /** Block height range to scan for trimmable online accounts signatures.
@@ -521,6 +533,22 @@ public class Settings { return this.atStatesTrimLimit; } + public boolean isPruningEnabled() { + return this.pruningEnabled; + } + + public int getPruneBlockLimit() { + return this.pruneBlockLimit; + } + + public long getAtStatesPruneInterval() { + return this.atStatesPruneInterval; + } + + public int getAtStatesPruneBatchSize() { + return this.atStatesPruneBatchSize; + } + public long getOnlineSignaturesTrimInterval() { return this.onlineSignaturesTrimInterval; } From bc1af126559289e96dee6fa2ef922c6ba7bc51f5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 22 Aug 2021 15:17:46 +0100 Subject: [PATCH 02/22] Prune all blocks up until the blockPruneLimit By default, this leaves only the last 1450 blocks in the database. Only applies when pruning mode is enabled. --- .../controller/pruning/BlockPruner.java | 86 +++++++++++++++++++ .../controller/pruning/PruneManager.java | 1 + .../qortal/repository/BlockRepository.java | 14 +++ .../hsqldb/HSQLDBBlockRepository.java | 47 ++++++++++ .../hsqldb/HSQLDBDatabaseUpdates.java | 1 + .../java/org/qortal/settings/Settings.java | 37 +++++--- 6 files changed, 176 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/qortal/controller/pruning/BlockPruner.java diff --git a/src/main/java/org/qortal/controller/pruning/BlockPruner.java b/src/main/java/org/qortal/controller/pruning/BlockPruner.java new file mode 100644 index 00000000..8ae25224 --- /dev/null +++ b/src/main/java/org/qortal/controller/pruning/BlockPruner.java @@ -0,0 +1,86 @@ +package org.qortal.controller.pruning; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.utils.NTP; + +public class BlockPruner implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(BlockPruner.class); + + @Override + public void run() { + Thread.currentThread().setName("Block pruner"); + + if (!Settings.getInstance().isPruningEnabled()) { + return; + } + + try (final Repository repository = RepositoryManager.getRepository()) { + int pruneStartHeight = repository.getBlockRepository().getBlockPruneHeight(); + + while (!Controller.isStopping()) { + repository.discardChanges(); + + Thread.sleep(Settings.getInstance().getBlockPruneInterval()); + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) + continue; + + // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages + if (Controller.getInstance().isSynchronizing()) + continue; + + // Prune all blocks up until our latest minus pruneBlockLimit + final int ourLatestHeight = chainTip.getHeight(); + final int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); + + int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize(); + int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); + + if (pruneStartHeight >= upperPruneHeight) { + continue; + } + + LOGGER.debug(String.format("Pruning blocks between %d and %d...", pruneStartHeight, upperPruneHeight)); + + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(pruneStartHeight, upperPruneHeight); + repository.saveChanges(); + + if (numBlocksPruned > 0) { + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.debug(() -> String.format("Pruned %d block%s between %d and %d", + numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""), + finalPruneStartHeight, upperPruneHeight)); + } else { + // Can we move onto next batch? + if (upperPrunableHeight > upperBatchHeight) { + pruneStartHeight = upperBatchHeight; + repository.getBlockRepository().setBlockPruneHeight(pruneStartHeight); + repository.saveChanges(); + + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.debug(() -> String.format("Bumping block base prune height to %d", finalPruneStartHeight)); + } + else { + // We've pruned up to the upper prunable height + // Back off for a while to save CPU for syncing + Thread.sleep(10*60*1000L); + } + } + } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to prune blocks: %s", e.getMessage())); + } catch (InterruptedException e) { + // Time to exit + } + } + +} diff --git a/src/main/java/org/qortal/controller/pruning/PruneManager.java b/src/main/java/org/qortal/controller/pruning/PruneManager.java index dcd7391d..66019d01 100644 --- a/src/main/java/org/qortal/controller/pruning/PruneManager.java +++ b/src/main/java/org/qortal/controller/pruning/PruneManager.java @@ -23,6 +23,7 @@ public class PruneManager { // Start individual pruning processes ExecutorService pruneExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory()); pruneExecutor.execute(new AtStatesPruner()); + pruneExecutor.execute(new BlockPruner()); } public static synchronized PruneManager getInstance() { diff --git a/src/main/java/org/qortal/repository/BlockRepository.java b/src/main/java/org/qortal/repository/BlockRepository.java index 78eba399..5ca61e66 100644 --- a/src/main/java/org/qortal/repository/BlockRepository.java +++ b/src/main/java/org/qortal/repository/BlockRepository.java @@ -166,6 +166,20 @@ public interface BlockRepository { */ public BlockData getDetachedBlockSignature(int startHeight) throws DataException; + + /** Returns height of first prunable block. */ + public int getBlockPruneHeight() throws DataException; + + /** Sets new base height for block pruning. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ + public void setBlockPruneHeight(int pruneHeight) throws DataException; + + /** Prunes full block data between passed heights. Returns number of pruned rows. */ + public int pruneBlocks(int minHeight, int maxHeight) throws DataException; + + /** * Saves block into repository. * diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java index b486e6a0..2f7e4ad2 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java @@ -509,6 +509,53 @@ public class HSQLDBBlockRepository implements BlockRepository { } } + + @Override + public int getBlockPruneHeight() throws DataException { + String sql = "SELECT block_prune_height FROM DatabaseInfo"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return 0; + + return resultSet.getInt(1); + } catch (SQLException e) { + throw new DataException("Unable to fetch block prune height from repository", e); + } + } + + @Override + public void setBlockPruneHeight(int pruneHeight) throws DataException { + // trimHeightsLock is to prevent concurrent update on DatabaseInfo + // that could result in "transaction rollback: serialization failure" + synchronized (this.repository.trimHeightsLock) { + String updateSql = "UPDATE DatabaseInfo SET block_prune_height = ?"; + + try { + this.repository.executeCheckedUpdate(updateSql, pruneHeight); + this.repository.saveChanges(); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to set block prune height in repository", e); + } + } + } + + @Override + public int pruneBlocks(int minHeight, int maxHeight) throws DataException { + // Don't prune the genesis block + if (minHeight <= 1) { + minHeight = 2; + } + + try { + return this.repository.delete("Blocks", "height BETWEEN ? AND ?", minHeight, maxHeight); + } catch (SQLException e) { + throw new DataException("Unable to prune blocks from repository", e); + } + } + + @Override public BlockData getDetachedBlockSignature(int startHeight) throws DataException { String sql = "SELECT " + BLOCK_DB_COLUMNS + " FROM Blocks " diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 94e753e8..d696351f 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -870,6 +870,7 @@ public class HSQLDBDatabaseUpdates { case 35: // Support for pruning stmt.execute("ALTER TABLE DatabaseInfo ADD AT_prune_height INT NOT NULL DEFAULT 0"); + stmt.execute("ALTER TABLE DatabaseInfo ADD block_prune_height INT NOT NULL DEFAULT 0"); break; default: diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index f63bdbb9..89a63ad1 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -103,23 +103,31 @@ public class Settings { /** Max number of AT states to trim in one go. */ private int atStatesTrimLimit = 4000; // records + /** How often to attempt online accounts signatures trimming (ms). */ + private long onlineSignaturesTrimInterval = 9876L; // milliseconds + /** Block height range to scan for trimmable online accounts signatures.
+ * This has a significant effect on execution time. */ + private int onlineSignaturesTrimBatchSize = 100; // blocks + + /** Whether we should prune old data to reduce database size * This prevents the node from being able to serve older blocks */ private boolean pruningEnabled = false; /** The amount of recent blocks we should keep when pruning */ - private int pruneBlockLimit = 1440; + private int pruneBlockLimit = 1450; /** How often to attempt AT state pruning (ms). */ private long atStatesPruneInterval = 3219L; // milliseconds - /** Block height range to scan for trimmable AT states.
+ /** Block height range to scan for prunable AT states.
* This has a significant effect on execution time. */ private int atStatesPruneBatchSize = 10; // blocks - /** How often to attempt online accounts signatures trimming (ms). */ - private long onlineSignaturesTrimInterval = 9876L; // milliseconds - /** Block height range to scan for trimmable online accounts signatures.
+ /** How often to attempt block pruning (ms). */ + private long blockPruneInterval = 3219L; // milliseconds + /** Block height range to scan for prunable blocks.
* This has a significant effect on execution time. */ - private int onlineSignaturesTrimBatchSize = 100; // blocks + private int blockPruneBatchSize = 10000; // blocks + // Peer-to-peer related private boolean isTestNet = false; @@ -533,6 +541,15 @@ public class Settings { return this.atStatesTrimLimit; } + public long getOnlineSignaturesTrimInterval() { + return this.onlineSignaturesTrimInterval; + } + + public int getOnlineSignaturesTrimBatchSize() { + return this.onlineSignaturesTrimBatchSize; + } + + public boolean isPruningEnabled() { return this.pruningEnabled; } @@ -549,12 +566,12 @@ public class Settings { return this.atStatesPruneBatchSize; } - public long getOnlineSignaturesTrimInterval() { - return this.onlineSignaturesTrimInterval; + public long getBlockPruneInterval() { + return this.blockPruneInterval; } - public int getOnlineSignaturesTrimBatchSize() { - return this.onlineSignaturesTrimBatchSize; + public int getBlockPruneBatchSize() { + return this.blockPruneBatchSize; } } From 209a9fa8c3ba25e111f6c113691e7a398e8fd129 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 22 Aug 2021 15:18:11 +0100 Subject: [PATCH 03/22] Rework of Blockchain.validate() to account for pruning mode. --- .../java/org/qortal/block/BlockChain.java | 49 ++++++++++++++----- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index e6b8db4e..15801193 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -506,28 +506,51 @@ public class BlockChain { * @throws SQLException */ public static void validate() throws DataException { - // Check first block is Genesis Block - if (!isGenesisBlockValid()) - rebuildBlockchain(); - try (final Repository repository = RepositoryManager.getRepository()) { + + boolean pruningEnabled = Settings.getInstance().isPruningEnabled(); + BlockData chainTip = repository.getBlockRepository().getLastBlock(); + boolean hasBlocks = (chainTip != null && chainTip.getHeight() > 1); + + if (pruningEnabled && hasBlocks) { + // Pruning is enabled and we have blocks, so it's possible that the genesis block has been pruned + // It's best not to validate it, and there's no real need to + } + else { + // Check first block is Genesis Block + if (!isGenesisBlockValid()) { + rebuildBlockchain(); + } + } + repository.checkConsistency(); - int startHeight = Math.max(repository.getBlockRepository().getBlockchainHeight() - 1440, 1); + // Set the number of blocks to validate based on the pruned state of the chain + // If pruned, subtract an extra 10 to allow room for error + int blocksToValidate = pruningEnabled ? Settings.getInstance().getPruneBlockLimit() - 10 : 1440; + int startHeight = Math.max(repository.getBlockRepository().getBlockchainHeight() - blocksToValidate, 1); BlockData detachedBlockData = repository.getBlockRepository().getDetachedBlockSignature(startHeight); if (detachedBlockData != null) { LOGGER.error(String.format("Block %d's reference does not match any block's signature", detachedBlockData.getHeight())); - // Wait for blockchain lock (whereas orphan() only tries to get lock) - ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - blockchainLock.lock(); - try { - LOGGER.info(String.format("Orphaning back to block %d", detachedBlockData.getHeight() - 1)); - orphan(detachedBlockData.getHeight() - 1); - } finally { - blockchainLock.unlock(); + // Orphan if we aren't a pruning node + if (!Settings.getInstance().isPruningEnabled()) { + + // Wait for blockchain lock (whereas orphan() only tries to get lock) + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + blockchainLock.lock(); + try { + LOGGER.info(String.format("Orphaning back to block %d", detachedBlockData.getHeight() - 1)); + orphan(detachedBlockData.getHeight() - 1); + } finally { + blockchainLock.unlock(); + } + } + else { + LOGGER.error(String.format("Not orphaning because we are in pruning mode. You may be on an " + + "invalid chain and should consider bootstrapping or re-syncing from genesis.")); } } } From c8466a2e7a0766478467125bed70cad6f4269d94 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 22 Aug 2021 15:20:11 +0100 Subject: [PATCH 04/22] Updated AT states pruner as it previously relied on blocks being present in the db to make decisions. As a side effect, this now prunes ATs up the the pruneBlockLimit too, rather than keeping the last 35 days or so. Will review this later but I don't think we will need the missing ones. --- .../org/qortal/controller/pruning/AtStatesPruner.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java b/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java index 37f0cd74..4268f98c 100644 --- a/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java +++ b/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java @@ -49,12 +49,9 @@ public class AtStatesPruner implements Runnable { if (Controller.getInstance().isSynchronizing()) continue; - long currentPrunableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime(); - // We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks - long chainPrunableTimestamp = chainTip.getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime(); - - long upperPrunableTimestamp = Math.min(currentPrunableTimestamp, chainPrunableTimestamp); - int upperPrunableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperPrunableTimestamp); + // Prune AT states for all blocks up until our latest minus pruneBlockLimit + final int ourLatestHeight = chainTip.getHeight(); + final int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); int upperBatchHeight = pruneStartHeight + Settings.getInstance().getAtStatesPruneBatchSize(); int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); From 22efaccd4aca79f397551804f610d913465cf1a1 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 22 Aug 2021 15:20:28 +0100 Subject: [PATCH 05/22] Fixed NPE introduced in earlier commit. --- src/main/java/org/qortal/controller/Controller.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 2b0a6b8f..26fc7fcc 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1415,10 +1415,12 @@ public class Controller extends Thread { BlockData blockData = repository.getBlockRepository().fromReference(parentSignature); - if (PruneManager.getInstance().isBlockPruned(blockData.getHeight(), repository)) { - // If this request contains a pruned block, we likely only have partial data, so best not to sent anything - // We always prune from the oldest first, so it's fine to just check the first block requested - blockData = null; + if (blockData != null) { + if (PruneManager.getInstance().isBlockPruned(blockData.getHeight(), repository)) { + // If this request contains a pruned block, we likely only have partial data, so best not to sent anything + // We always prune from the oldest first, so it's fine to just check the first block requested + blockData = null; + } } while (blockData != null && blockSummaries.size() < numberRequested) { From f5910ab95071684b8683b35815ecbaa19e2638fb Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 22 Aug 2021 15:21:18 +0100 Subject: [PATCH 06/22] Break out of the AT pruning inner loops if we're stopping the app. --- .../qortal/repository/hsqldb/HSQLDBATRepository.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index d5929311..0d4d2923 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -727,6 +727,11 @@ public class HSQLDBATRepository implements ATRepository { for (int height=minHeight; height atAddresses = new ArrayList<>(); String updateSql = "SELECT AT_address FROM LatestATStates WHERE height = ?"; @@ -746,6 +751,11 @@ public class HSQLDBATRepository implements ATRepository { for (ATStateData atState : atStates) { //LOGGER.info("Found atState {} at height {}", atState.getATAddress(), atState.getHeight()); + // Give up if we're stopping + if (Controller.isStopping()) { + return deletedCount; + } + if (atAddresses.contains(atState.getATAddress())) { // We don't want to delete this AT state because it is still active LOGGER.info("Skipping atState {} at height {}", atState.getATAddress(), atState.getHeight()); From 5127f9442397ac068ab6b2944c73886dc074b73b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 23 Aug 2021 21:17:51 +0100 Subject: [PATCH 07/22] Added bulk pruning phase on node startup the first time that pruning mode is enabled. When switching from a full node to a pruning node, we need to delete most of the database contents. If we do this entirely as a background process, it is very slow and can interfere with syncing. However, if we take the approach of transferring only the necessary rows to a new table and then deleting the original table, this makes the process much faster. It was taking several days to delete the AT states in the background, but only a couple of minutes to copy them to a new table. The trade off is that we have to go through a form of "reshape" when starting the app for the first time after enabling pruning mode. But given that this is an opt-in mode, I don't think it will be a problem. Once the pruning is complete, it automatically performs a CHECKPOINT DEFRAG in order to shrink the database file size down to a fraction of what it was before. From this point, the original background process will run, but can be dialled right down so not to interfere with syncing. --- .../org/qortal/controller/Controller.java | 1 + .../qortal/repository/RepositoryManager.java | 24 ++ .../hsqldb/HSQLDBDatabasePruning.java | 217 ++++++++++++++++++ 3 files changed, 242 insertions(+) create mode 100644 src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 26fc7fcc..a34a5d81 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -416,6 +416,7 @@ public class Controller extends Thread { try { RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl()); RepositoryManager.setRepositoryFactory(repositoryFactory); + RepositoryManager.prune(); } catch (DataException e) { // If exception has no cause then repository is in use by some other process. if (e.getCause() == null) { diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index df578888..5e9c71c2 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -1,8 +1,14 @@ package org.qortal.repository; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.repository.hsqldb.HSQLDBDatabasePruning; +import org.qortal.settings.Settings; + import java.sql.SQLException; public abstract class RepositoryManager { + private static final Logger LOGGER = LogManager.getLogger(RepositoryManager.class); private static RepositoryFactory repositoryFactory = null; @@ -51,6 +57,24 @@ public abstract class RepositoryManager { } } + public static void prune() { + // Bulk prune the database the first time we use pruning mode + if (Settings.getInstance().isPruningEnabled()) { + try { + boolean prunedATStates = HSQLDBDatabasePruning.pruneATStates(); + boolean prunedBlocks = HSQLDBDatabasePruning.pruneBlocks(); + + // Perform repository maintenance to shrink the db size down + if (prunedATStates && prunedBlocks) { + HSQLDBDatabasePruning.performMaintenance(); + } + + } catch (SQLException | DataException e) { + LOGGER.info("Unable to bulk prune AT states. The database may have been left in an inconsistent state."); + } + } + } + public static void setRequestedCheckpoint(Boolean quick) { quickCheckpointRequested = quick; } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java new file mode 100644 index 00000000..6dc50647 --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java @@ -0,0 +1,217 @@ +package org.qortal.repository.hsqldb; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * + * When switching from a full node to a pruning node, we need to delete most of the database contents. + * If we do this entirely as a background process, it is very slow and can interfere with syncing. + * However, if we take the approach of transferring only the necessary rows to a new table and then + * deleting the original table, this makes the process much faster. It was taking several days to + * delete the AT states in the background, but only a couple of minutes to copy them to a new table. + * + * The trade off is that we have to go through a form of "reshape" when starting the app for the first + * time after enabling pruning mode. But given that this is an opt-in mode, I don't think it will be + * a problem. + * + * Once the pruning is complete, it automatically performs a CHECKPOINT DEFRAG in order to + * shrink the database file size down to a fraction of what it was before. + * + * From this point, the original background process will run, but can be dialled right down so not + * to interfere with syncing. + * + */ + + +public class HSQLDBDatabasePruning { + + private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabasePruning.class); + + public static boolean pruneATStates() throws SQLException, DataException { + try (final HSQLDBRepository repository = (HSQLDBRepository)RepositoryManager.getRepository()) { + + // Only bulk prune AT states if we have never done so before + int pruneHeight = repository.getATRepository().getAtPruneHeight(); + if (pruneHeight > 0) { + // Already pruned AT states + return false; + } + + LOGGER.info("Starting bulk prune of AT states - this process could take a while... (approx. 2 mins on high spec)"); + + // Create new AT-states table to hold smaller dataset + repository.executeCheckedUpdate("DROP TABLE IF EXISTS ATStatesNew"); + repository.executeCheckedUpdate("CREATE TABLE ATStatesNew (" + + "AT_address QortalAddress, height INTEGER NOT NULL, state_hash ATStateHash NOT NULL, " + + "fees QortalAmount NOT NULL, is_initial BOOLEAN NOT NULL, sleep_until_message_timestamp BIGINT, " + + "PRIMARY KEY (AT_address, height), " + + "FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)"); + repository.executeCheckedUpdate("SET TABLE ATStatesNew NEW SPACE"); + repository.executeCheckedUpdate("CHECKPOINT"); + + + // Find our latest block + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + if (latestBlock == null) { + LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning"); + return false; + } + + // Calculate some constants for later use + final int blockchainHeight = latestBlock.getHeight(); + final int maximumBlockToTrim = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + final int startHeight = maximumBlockToTrim; + final int endHeight = blockchainHeight; + final int blockStep = 10000; + + // Loop through all the LatestATStates and copy them to the new table + LOGGER.info("Copying AT states..."); + for (int height = 0; height < endHeight; height += blockStep) { + //LOGGER.info(String.format("Copying AT states between %d and %d...", height, height + blockStep - 1)); + + String sql = "SELECT height, AT_address FROM LatestATStates WHERE height BETWEEN ? AND ?"; + try (ResultSet latestAtStatesResultSet = repository.checkedExecute(sql, height, height + blockStep - 1)) { + if (latestAtStatesResultSet != null) { + do { + int latestAtHeight = latestAtStatesResultSet.getInt(1); + String latestAtAddress = latestAtStatesResultSet.getString(2); + + // Copy this latest ATState to the new table + //LOGGER.info(String.format("Copying AT %s at height %d...", latestAtAddress, latestAtHeight)); + try { + String updateSql = "INSERT INTO ATStatesNew (" + + "SELECT AT_address, height, state_hash, fees, is_initial, sleep_until_message_timestamp " + + "FROM ATStates " + + "WHERE height = ? AND AT_address = ?)"; + repository.executeCheckedUpdate(updateSql, latestAtHeight, latestAtAddress); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to copy ATStates", e); + } + + if (height >= startHeight) { + // Now copy this AT states for each recent block it is present in + for (int i = startHeight; i < endHeight; i++) { + if (latestAtHeight < i) { + // This AT finished before this block so there is nothing to copy + continue; + } + + //LOGGER.info(String.format("Copying recent AT %s at height %d...", latestAtAddress, i)); + try { + // Copy each LatestATState to the new table + String updateSql = "INSERT IGNORE INTO ATStatesNew (" + + "SELECT AT_address, height, state_hash, fees, is_initial, sleep_until_message_timestamp " + + "FROM ATStates " + + "WHERE height = ? AND AT_address = ?)"; + repository.executeCheckedUpdate(updateSql, i, latestAtAddress); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to copy ATStates", e); + } + } + } + + } while (latestAtStatesResultSet.next()); + } + } catch (SQLException e) { + throw new DataException("Unable to copy AT states", e); + } + } + + repository.saveChanges(); + + // Add a height index + LOGGER.info("Rebuilding AT states height index in repository"); + repository.executeCheckedUpdate("CREATE INDEX IF NOT EXISTS ATStatesHeightIndex ON ATStatesNew (height)"); + repository.executeCheckedUpdate("CHECKPOINT"); + + // Finally, drop the original table and rename + LOGGER.info("Deleting old AT states..."); + repository.executeCheckedUpdate("DROP TABLE ATStates"); + repository.executeCheckedUpdate("ALTER TABLE ATStatesNew RENAME TO ATStates"); + repository.executeCheckedUpdate("CHECKPOINT"); + + // Update the prune height + repository.getATRepository().setAtPruneHeight(maximumBlockToTrim); + repository.saveChanges(); + + repository.executeCheckedUpdate("CHECKPOINT"); + + return true; + } + } + + public static boolean pruneBlocks() throws SQLException, DataException { + try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) { + + // Only bulk prune AT states if we have never done so before + int pruneHeight = repository.getBlockRepository().getBlockPruneHeight(); + if (pruneHeight > 0) { + // Already pruned blocks + return false; + } + + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + if (latestBlock == null) { + LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning"); + return false; + } + final int blockchainHeight = latestBlock.getHeight(); + final int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + int pruneStartHeight = 0; + + LOGGER.info("Starting bulk prune of blocks - this process could take a while... (approx. 10 mins on high spec)"); + + while (pruneStartHeight < upperPrunableHeight) { + // Prune all blocks up until our latest minus pruneBlockLimit + + int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize(); + int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); + + LOGGER.info(String.format("Pruning blocks between %d and %d...", pruneStartHeight, upperPruneHeight)); + + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(pruneStartHeight, upperPruneHeight); + repository.saveChanges(); + + if (numBlocksPruned > 0) { + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.info(() -> String.format("Pruned %d block%s between %d and %d", + numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""), + finalPruneStartHeight, upperPruneHeight)); + } else { + // Can we move onto next batch? + if (upperPrunableHeight > upperBatchHeight) { + pruneStartHeight = upperBatchHeight; + repository.getBlockRepository().setBlockPruneHeight(pruneStartHeight); + repository.saveChanges(); + + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.debug(() -> String.format("Bumping block base prune height to %d", finalPruneStartHeight)); + } + else { + // We've finished pruning + break; + } + } + } + + return true; + } + } + + public static void performMaintenance() throws SQLException, DataException { + try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) { + repository.performPeriodicMaintenance(); + } + } + +} From ca1379d9f8a05341489efabff50778fa60ed91e1 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 25 Aug 2021 18:50:41 +0100 Subject: [PATCH 08/22] Unified the code to build the LatestATStates table, as it's now used by more than one class. Note - the rebuildLatestAtStates() must never be used by two different classes at the same time, or AT states could be incorrectly deleted. It is okay at the moment as we don't run the AT states trimmer and pruner in the same app session. However we should probably synchronize this method so that we don't accidentally call it from two places in the future. --- .../qortal/controller/AtStatesTrimmer.java | 5 +- .../org/qortal/controller/Controller.java | 13 +--- .../controller/pruning/AtStatesPruner.java | 24 +++---- .../controller/pruning/PruneManager.java | 56 ++++++++++++---- .../org/qortal/repository/ATRepository.java | 11 ++-- .../repository/hsqldb/HSQLDBATRepository.java | 64 +++++++++---------- 6 files changed, 94 insertions(+), 79 deletions(-) diff --git a/src/main/java/org/qortal/controller/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/AtStatesTrimmer.java index 78539813..4b08e5ca 100644 --- a/src/main/java/org/qortal/controller/AtStatesTrimmer.java +++ b/src/main/java/org/qortal/controller/AtStatesTrimmer.java @@ -21,9 +21,8 @@ public class AtStatesTrimmer implements Runnable { try (final Repository repository = RepositoryManager.getRepository()) { int trimStartHeight = repository.getATRepository().getAtTrimHeight(); - repository.getATRepository().prepareForAtStateTrimming(); + repository.getATRepository().rebuildLatestAtStates(); repository.saveChanges(); - PruneManager.getInstance().setBuiltLatestATStates(true); while (!Controller.isStopping()) { repository.discardChanges(); @@ -64,7 +63,7 @@ public class AtStatesTrimmer implements Runnable { if (upperTrimmableHeight > upperBatchHeight) { trimStartHeight = upperBatchHeight; repository.getATRepository().setAtTrimHeight(trimStartHeight); - repository.getATRepository().prepareForAtStateTrimming(); + repository.getATRepository().rebuildLatestAtStates(); repository.saveChanges(); final int finalTrimStartHeight = trimStartHeight; diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index a34a5d81..d8f706fd 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -512,9 +512,8 @@ public class Controller extends Thread { final long repositoryBackupInterval = Settings.getInstance().getRepositoryBackupInterval(); final long repositoryCheckpointInterval = Settings.getInstance().getRepositoryCheckpointInterval(); - ExecutorService trimExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory()); - trimExecutor.execute(new AtStatesTrimmer()); - trimExecutor.execute(new OnlineAccountsSignaturesTrimmer()); + // Start executor service for trimming or pruning + PruneManager.getInstance().start(); try { while (!isStopping) { @@ -599,13 +598,7 @@ public class Controller extends Thread { Thread.interrupted(); // Fall-through to exit } finally { - trimExecutor.shutdownNow(); - - try { - trimExecutor.awaitTermination(2L, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // We tried... - } + PruneManager.getInstance().stop(); } } diff --git a/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java b/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java index 4268f98c..66325e88 100644 --- a/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java +++ b/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java @@ -25,22 +25,14 @@ public class AtStatesPruner implements Runnable { try (final Repository repository = RepositoryManager.getRepository()) { int pruneStartHeight = repository.getATRepository().getAtPruneHeight(); - // repository.getATRepository().prepareForAtStatePruning(); - // repository.saveChanges(); + repository.getATRepository().rebuildLatestAtStates(); + repository.saveChanges(); while (!Controller.isStopping()) { repository.discardChanges(); Thread.sleep(Settings.getInstance().getAtStatesPruneInterval()); - if (PruneManager.getInstance().getBuiltLatestATStates() == false) { - // Wait for latest AT states table to be built first - // This has a dependency on the AtStatesTrimmer running, - // which should be okay, given that it isn't something - // is disabled in normal operation. - continue; - } - BlockData chainTip = Controller.getInstance().getChainTip(); if (chainTip == null || NTP.getTime() == null) continue; @@ -63,8 +55,11 @@ public class AtStatesPruner implements Runnable { int numAtStatesPruned = repository.getATRepository().pruneAtStates(pruneStartHeight, upperPruneHeight); repository.saveChanges(); + int numAtStateDataRowsTrimmed = repository.getATRepository().trimAtStates( + pruneStartHeight, upperPruneHeight, Settings.getInstance().getAtStatesTrimLimit()); + repository.saveChanges(); - if (numAtStatesPruned > 0) { + if (numAtStatesPruned > 0 || numAtStateDataRowsTrimmed > 0) { final int finalPruneStartHeight = pruneStartHeight; LOGGER.debug(() -> String.format("Pruned %d AT state%s between blocks %d and %d", numAtStatesPruned, (numAtStatesPruned != 1 ? "s" : ""), @@ -74,12 +69,17 @@ public class AtStatesPruner implements Runnable { if (upperPrunableHeight > upperBatchHeight) { pruneStartHeight = upperBatchHeight; repository.getATRepository().setAtPruneHeight(pruneStartHeight); - repository.getATRepository().prepareForAtStatePruning(); + repository.getATRepository().rebuildLatestAtStates(); repository.saveChanges(); final int finalPruneStartHeight = pruneStartHeight; LOGGER.debug(() -> String.format("Bumping AT state base prune height to %d", finalPruneStartHeight)); } + else { + // We've pruned up to the upper prunable height + // Back off for a while to save CPU for syncing + Thread.sleep(5*60*1000L); + } } } } catch (DataException e) { diff --git a/src/main/java/org/qortal/controller/pruning/PruneManager.java b/src/main/java/org/qortal/controller/pruning/PruneManager.java index 66019d01..b733833b 100644 --- a/src/main/java/org/qortal/controller/pruning/PruneManager.java +++ b/src/main/java/org/qortal/controller/pruning/PruneManager.java @@ -1,7 +1,9 @@ package org.qortal.controller.pruning; +import org.qortal.controller.AtStatesTrimmer; import org.qortal.controller.Controller; +import org.qortal.controller.OnlineAccountsSignaturesTrimmer; import org.qortal.data.block.BlockData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -10,6 +12,7 @@ import org.qortal.utils.DaemonThreadFactory; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; public class PruneManager { @@ -17,13 +20,11 @@ public class PruneManager { private boolean pruningEnabled = Settings.getInstance().isPruningEnabled(); private int pruneBlockLimit = Settings.getInstance().getPruneBlockLimit(); - private boolean builtLatestATStates = false; + + private ExecutorService executorService; private PruneManager() { - // Start individual pruning processes - ExecutorService pruneExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory()); - pruneExecutor.execute(new AtStatesPruner()); - pruneExecutor.execute(new BlockPruner()); + } public static synchronized PruneManager getInstance() { @@ -33,6 +34,42 @@ public class PruneManager { return instance; } + public void start() { + this.executorService = Executors.newCachedThreadPool(new DaemonThreadFactory()); + + // Don't allow both the pruner and the trimmer to run at the same time. + // In pruning mode, we are already deleting far more than we would when trimming. + // In non-pruning mode, we still need to trim to keep the non-essential data + // out of the database. There isn't a case where both are needed at once. + // If we ever do need to enable both at once, be very careful with the AT state + // trimming, since both currently rely on having exclusive access to the + // prepareForAtStateTrimming() method. For both trimming and pruning to take place + // at once, we would need to synchronize this method in a way that both can't + // call it at the same time, as otherwise active ATs would be pruned/trimmed when + // they should have been kept. + + if (Settings.getInstance().isPruningEnabled()) { + // Pruning enabled - start the pruning processes + this.executorService.execute(new AtStatesPruner()); + this.executorService.execute(new BlockPruner()); + } + else { + // Pruning disabled - use trimming instead + this.executorService.execute(new AtStatesTrimmer()); + this.executorService.execute(new OnlineAccountsSignaturesTrimmer()); + } + } + + public void stop() { + this.executorService.shutdownNow(); + + try { + this.executorService.awaitTermination(2L, TimeUnit.SECONDS); + } catch (InterruptedException e) { + // We tried... + } + } + public boolean isBlockPruned(int height, Repository repository) throws DataException { if (!this.pruningEnabled) { return false; @@ -49,13 +86,4 @@ public class PruneManager { return (height < latestUnprunedHeight); } - - public void setBuiltLatestATStates(boolean builtLatestATStates) { - this.builtLatestATStates = builtLatestATStates; - } - - public boolean getBuiltLatestATStates() { - return this.builtLatestATStates; - } - } diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 6cec0839..74fb19ab 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -112,6 +112,11 @@ public interface ATRepository { */ public List getBlockATStatesAtHeight(int height) throws DataException; + + /** Rebuild the latest AT states cache, necessary for AT state trimming/pruning. */ + public void rebuildLatestAtStates() throws DataException; + + /** Returns height of first trimmable AT state. */ public int getAtTrimHeight() throws DataException; @@ -121,9 +126,6 @@ public interface ATRepository { */ public void setAtTrimHeight(int trimHeight) throws DataException; - /** Hook to allow repository to prepare/cache info for AT state trimming. */ - public void prepareForAtStateTrimming() throws DataException; - /** Trims full AT state data between passed heights. Returns number of trimmed rows. */ public int trimAtStates(int minHeight, int maxHeight, int limit) throws DataException; @@ -137,9 +139,6 @@ public interface ATRepository { */ public void setAtPruneHeight(int pruneHeight) throws DataException; - /** Hook to allow repository to prepare/cache info for AT state pruning. */ - public void prepareForAtStatePruning() throws DataException; - /** Prunes full AT state data between passed heights. Returns number of pruned rows. */ public int pruneAtStates(int minHeight, int maxHeight) throws DataException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index 0d4d2923..1921661c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -8,7 +8,7 @@ import java.util.Set; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.data.account.AccountData; +import org.qortal.controller.Controller; import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.repository.ATRepository; @@ -601,6 +601,35 @@ public class HSQLDBATRepository implements ATRepository { return atStates; } + + @Override + public void rebuildLatestAtStates() throws DataException { + // Rebuild cache of latest AT states that we can't trim + String deleteSql = "DELETE FROM LatestATStates"; + try { + this.repository.executeCheckedUpdate(deleteSql); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to delete temporary latest AT states cache from repository", e); + } + + String insertSql = "INSERT INTO LatestATStates (" + + "SELECT AT_address, height FROM ATs " + + "CROSS JOIN LATERAL(" + + "SELECT height FROM ATStates " + + "WHERE ATStates.AT_address = ATs.AT_address " + + "ORDER BY AT_address DESC, height DESC LIMIT 1" + + ") " + + ")"; + try { + this.repository.executeCheckedUpdate(insertSql); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to populate temporary latest AT states cache in repository", e); + } + } + + @Override public int getAtTrimHeight() throws DataException { String sql = "SELECT AT_trim_height FROM DatabaseInfo"; @@ -632,33 +661,6 @@ public class HSQLDBATRepository implements ATRepository { } } - @Override - public void prepareForAtStateTrimming() throws DataException { - // Rebuild cache of latest AT states that we can't trim - String deleteSql = "DELETE FROM LatestATStates"; - try { - this.repository.executeCheckedUpdate(deleteSql); - } catch (SQLException e) { - repository.examineException(e); - throw new DataException("Unable to delete temporary latest AT states cache from repository", e); - } - - String insertSql = "INSERT INTO LatestATStates (" - + "SELECT AT_address, height FROM ATs " - + "CROSS JOIN LATERAL(" - + "SELECT height FROM ATStates " - + "WHERE ATStates.AT_address = ATs.AT_address " - + "ORDER BY AT_address DESC, height DESC LIMIT 1" - + ") " - + ")"; - try { - this.repository.executeCheckedUpdate(insertSql); - } catch (SQLException e) { - repository.examineException(e); - throw new DataException("Unable to populate temporary latest AT states cache in repository", e); - } - } - @Override public int trimAtStates(int minHeight, int maxHeight, int limit) throws DataException { if (minHeight >= maxHeight) @@ -715,12 +717,6 @@ public class HSQLDBATRepository implements ATRepository { } } - @Override - public void prepareForAtStatePruning() throws DataException { - // Use LatestATStates table that was already built by AtStatesTrimmer - // The AtStatesPruner class checks that this process has completed first - } - @Override public int pruneAtStates(int minHeight, int maxHeight) throws DataException { int deletedCount = 0; From ff841c28e39f1618532dae2de84f3502d0b8a90e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 25 Aug 2021 18:51:45 +0100 Subject: [PATCH 09/22] Updated tests to use the renamed method. --- src/test/java/org/qortal/test/at/AtRepositoryTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/qortal/test/at/AtRepositoryTests.java b/src/test/java/org/qortal/test/at/AtRepositoryTests.java index c7dfa423..0b302435 100644 --- a/src/test/java/org/qortal/test/at/AtRepositoryTests.java +++ b/src/test/java/org/qortal/test/at/AtRepositoryTests.java @@ -75,7 +75,7 @@ public class AtRepositoryTests extends Common { Integer testHeight = maxHeight - 2; // Trim AT state data - repository.getATRepository().prepareForAtStateTrimming(); + repository.getATRepository().rebuildLatestAtStates(); repository.getATRepository().trimAtStates(2, maxHeight, 1000); ATStateData atStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight); @@ -129,7 +129,7 @@ public class AtRepositoryTests extends Common { Integer testHeight = blockchainHeight; // Trim AT state data - repository.getATRepository().prepareForAtStateTrimming(); + repository.getATRepository().rebuildLatestAtStates(); // COMMIT to check latest AT states persist / TEMPORARY table interaction repository.saveChanges(); @@ -280,7 +280,7 @@ public class AtRepositoryTests extends Common { Integer testHeight = maxHeight - 2; // Trim AT state data - repository.getATRepository().prepareForAtStateTrimming(); + repository.getATRepository().rebuildLatestAtStates(); repository.getATRepository().trimAtStates(2, maxHeight, 1000); List atStates = repository.getATRepository().getBlockATStatesAtHeight(testHeight); From cd9d9b31ef62d886a044d29e4b615fe19264eb3d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 25 Aug 2021 18:53:25 +0100 Subject: [PATCH 10/22] Prune ATStatesData as well as the ATStates when switching to pruning mode. --- .../hsqldb/HSQLDBDatabasePruning.java | 67 ++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java index 6dc50647..ba170bf6 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java @@ -2,6 +2,7 @@ package org.qortal.repository.hsqldb; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; import org.qortal.data.block.BlockData; import org.qortal.repository.DataException; import org.qortal.repository.RepositoryManager; @@ -146,6 +147,70 @@ public class HSQLDBDatabasePruning { repository.executeCheckedUpdate("CHECKPOINT"); + // Now prune/trim the ATStatesData, as this currently goes back over a month + return HSQLDBDatabasePruning.pruneATStateData(); + } + } + + /* + * Bulk prune ATStatesData to catch up with the now pruned ATStates table + * This uses the existing AT States trimming code but with a much higher end block + */ + private static boolean pruneATStateData() throws SQLException, DataException { + try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) { + + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + if (latestBlock == null) { + LOGGER.info("Unable to determine blockchain height, necessary for bulk ATStatesData pruning"); + return false; + } + final int blockchainHeight = latestBlock.getHeight(); + final int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + // ATStateData is already trimmed - so carry on from where we left off in the past + int pruneStartHeight = repository.getATRepository().getAtTrimHeight(); + + LOGGER.info("Starting bulk prune of AT states data - this process could take a while... (approx. 3 mins on high spec)"); + + while (pruneStartHeight < upperPrunableHeight) { + // Prune all AT state data up until our latest minus pruneBlockLimit + + if (Controller.isStopping()) { + return false; + } + + // Override batch size in the settings because this is a one-off process + final int batchSize = 1000; + final int rowLimitPerBatch = 50000; + int upperBatchHeight = pruneStartHeight + batchSize; + int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); + + LOGGER.trace(String.format("Pruning AT states data between %d and %d...", pruneStartHeight, upperPruneHeight)); + + int numATStatesPruned = repository.getATRepository().trimAtStates(pruneStartHeight, upperPruneHeight, rowLimitPerBatch); + repository.saveChanges(); + + if (numATStatesPruned > 0) { + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.trace(() -> String.format("Pruned %d AT states data rows between blocks %d and %d", + numATStatesPruned, finalPruneStartHeight, upperPruneHeight)); + } else { + // Can we move onto next batch? + if (upperPrunableHeight > upperBatchHeight) { + pruneStartHeight = upperBatchHeight; + repository.getATRepository().setAtTrimHeight(pruneStartHeight); + // No need to rebuild the latest AT states as we aren't currently synchronizing + repository.saveChanges(); + + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.debug(() -> String.format("Bumping AT states trim height to %d", finalPruneStartHeight)); + } + else { + // We've finished pruning + break; + } + } + } + return true; } } @@ -169,7 +234,7 @@ public class HSQLDBDatabasePruning { final int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); int pruneStartHeight = 0; - LOGGER.info("Starting bulk prune of blocks - this process could take a while... (approx. 10 mins on high spec)"); + LOGGER.info("Starting bulk prune of blocks - this process could take a while... (approx. 5 mins on high spec)"); while (pruneStartHeight < upperPrunableHeight) { // Prune all blocks up until our latest minus pruneBlockLimit From 9056cb70261303a3a4992a8f8190f26e75fa9bf8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 25 Aug 2021 18:54:10 +0100 Subject: [PATCH 11/22] Increased atStatesPruneBatchSize from 10 to 25. --- src/main/java/org/qortal/settings/Settings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 89a63ad1..84eeb3a2 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -120,7 +120,7 @@ public class Settings { private long atStatesPruneInterval = 3219L; // milliseconds /** Block height range to scan for prunable AT states.
* This has a significant effect on execution time. */ - private int atStatesPruneBatchSize = 10; // blocks + private int atStatesPruneBatchSize = 25; // blocks /** How often to attempt block pruning (ms). */ private long blockPruneInterval = 3219L; // milliseconds From 2479f2d65d1198049cdd59aa42856971096dc049 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 25 Aug 2021 18:57:04 +0100 Subject: [PATCH 12/22] Moved trimming and pruning classes into a single package (org.qortal.controller.repository) --- src/main/java/org/qortal/controller/Controller.java | 4 +--- .../controller/{pruning => repository}/AtStatesPruner.java | 2 +- .../qortal/controller/{ => repository}/AtStatesTrimmer.java | 4 ++-- .../controller/{pruning => repository}/BlockPruner.java | 2 +- .../{ => repository}/OnlineAccountsSignaturesTrimmer.java | 3 ++- .../controller/{pruning => repository}/PruneManager.java | 4 +--- 6 files changed, 8 insertions(+), 11 deletions(-) rename src/main/java/org/qortal/controller/{pruning => repository}/AtStatesPruner.java (98%) rename src/main/java/org/qortal/controller/{ => repository}/AtStatesTrimmer.java (97%) rename src/main/java/org/qortal/controller/{pruning => repository}/BlockPruner.java (98%) rename src/main/java/org/qortal/controller/{ => repository}/OnlineAccountsSignaturesTrimmer.java (97%) rename src/main/java/org/qortal/controller/{pruning => repository}/PruneManager.java (95%) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index d8f706fd..f9d48c70 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -24,7 +24,6 @@ import java.util.Properties; import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; @@ -46,7 +45,7 @@ import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.controller.Synchronizer.SynchronizationResult; -import org.qortal.controller.pruning.PruneManager; +import org.qortal.controller.repository.PruneManager; import org.qortal.controller.tradebot.TradeBot; import org.qortal.crypto.Crypto; import org.qortal.data.account.MintingAccountData; @@ -96,7 +95,6 @@ import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transaction.Transaction.ValidationResult; import org.qortal.utils.Base58; import org.qortal.utils.ByteArray; -import org.qortal.utils.DaemonThreadFactory; import org.qortal.utils.NTP; import org.qortal.utils.Triple; diff --git a/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java similarity index 98% rename from src/main/java/org/qortal/controller/pruning/AtStatesPruner.java rename to src/main/java/org/qortal/controller/repository/AtStatesPruner.java index 66325e88..30d7f136 100644 --- a/src/main/java/org/qortal/controller/pruning/AtStatesPruner.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java @@ -1,4 +1,4 @@ -package org.qortal.controller.pruning; +package org.qortal.controller.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/qortal/controller/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java similarity index 97% rename from src/main/java/org/qortal/controller/AtStatesTrimmer.java rename to src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java index 4b08e5ca..ed02ee47 100644 --- a/src/main/java/org/qortal/controller/AtStatesTrimmer.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java @@ -1,8 +1,8 @@ -package org.qortal.controller; +package org.qortal.controller.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.controller.pruning.PruneManager; +import org.qortal.controller.Controller; import org.qortal.data.block.BlockData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; diff --git a/src/main/java/org/qortal/controller/pruning/BlockPruner.java b/src/main/java/org/qortal/controller/repository/BlockPruner.java similarity index 98% rename from src/main/java/org/qortal/controller/pruning/BlockPruner.java rename to src/main/java/org/qortal/controller/repository/BlockPruner.java index 8ae25224..6d3180a8 100644 --- a/src/main/java/org/qortal/controller/pruning/BlockPruner.java +++ b/src/main/java/org/qortal/controller/repository/BlockPruner.java @@ -1,4 +1,4 @@ -package org.qortal.controller.pruning; +package org.qortal.controller.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/qortal/controller/OnlineAccountsSignaturesTrimmer.java b/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java similarity index 97% rename from src/main/java/org/qortal/controller/OnlineAccountsSignaturesTrimmer.java rename to src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java index b32a2b06..c7f248d5 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsSignaturesTrimmer.java +++ b/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java @@ -1,8 +1,9 @@ -package org.qortal.controller; +package org.qortal.controller.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.block.BlockChain; +import org.qortal.controller.Controller; import org.qortal.data.block.BlockData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; diff --git a/src/main/java/org/qortal/controller/pruning/PruneManager.java b/src/main/java/org/qortal/controller/repository/PruneManager.java similarity index 95% rename from src/main/java/org/qortal/controller/pruning/PruneManager.java rename to src/main/java/org/qortal/controller/repository/PruneManager.java index b733833b..5f92c75d 100644 --- a/src/main/java/org/qortal/controller/pruning/PruneManager.java +++ b/src/main/java/org/qortal/controller/repository/PruneManager.java @@ -1,9 +1,7 @@ -package org.qortal.controller.pruning; +package org.qortal.controller.repository; -import org.qortal.controller.AtStatesTrimmer; import org.qortal.controller.Controller; -import org.qortal.controller.OnlineAccountsSignaturesTrimmer; import org.qortal.data.block.BlockData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; From 9973fe4326aa09fcf9d6222ab67eb7c4e73d003f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 28 Aug 2021 11:00:49 +0100 Subject: [PATCH 13/22] Synchronized LatestATStates, to make rebuildLatestAtStates() thread safe. --- .../repository/hsqldb/HSQLDBATRepository.java | 160 ++++++++++-------- .../repository/hsqldb/HSQLDBRepository.java | 1 + 2 files changed, 90 insertions(+), 71 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index 1921661c..522fafb7 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -604,28 +604,34 @@ public class HSQLDBATRepository implements ATRepository { @Override public void rebuildLatestAtStates() throws DataException { - // Rebuild cache of latest AT states that we can't trim - String deleteSql = "DELETE FROM LatestATStates"; - try { - this.repository.executeCheckedUpdate(deleteSql); - } catch (SQLException e) { - repository.examineException(e); - throw new DataException("Unable to delete temporary latest AT states cache from repository", e); - } + // latestATStatesLock is to prevent concurrent updates on LatestATStates + // that could result in one process using a partial or empty dataset + // because it was in the process of being rebuilt by another thread + synchronized (this.repository.latestATStatesLock) { - String insertSql = "INSERT INTO LatestATStates (" - + "SELECT AT_address, height FROM ATs " - + "CROSS JOIN LATERAL(" - + "SELECT height FROM ATStates " - + "WHERE ATStates.AT_address = ATs.AT_address " - + "ORDER BY AT_address DESC, height DESC LIMIT 1" - + ") " - + ")"; - try { - this.repository.executeCheckedUpdate(insertSql); - } catch (SQLException e) { - repository.examineException(e); - throw new DataException("Unable to populate temporary latest AT states cache in repository", e); + // Rebuild cache of latest AT states that we can't trim + String deleteSql = "DELETE FROM LatestATStates"; + try { + this.repository.executeCheckedUpdate(deleteSql); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to delete temporary latest AT states cache from repository", e); + } + + String insertSql = "INSERT INTO LatestATStates (" + + "SELECT AT_address, height FROM ATs " + + "CROSS JOIN LATERAL(" + + "SELECT height FROM ATStates " + + "WHERE ATStates.AT_address = ATs.AT_address " + + "ORDER BY AT_address DESC, height DESC LIMIT 1" + + ") " + + ")"; + try { + this.repository.executeCheckedUpdate(insertSql); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to populate temporary latest AT states cache in repository", e); + } } } @@ -666,22 +672,28 @@ public class HSQLDBATRepository implements ATRepository { if (minHeight >= maxHeight) return 0; - // We're often called so no need to trim all states in one go. - // Limit updates to reduce CPU and memory load. - String sql = "DELETE FROM ATStatesData " - + "WHERE height BETWEEN ? AND ? " - + "AND NOT EXISTS(" + // latestATStatesLock is to prevent concurrent updates on LatestATStates + // that could result in one process using a partial or empty dataset + // because it was in the process of being rebuilt by another thread + synchronized (this.repository.latestATStatesLock) { + + // We're often called so no need to trim all states in one go. + // Limit updates to reduce CPU and memory load. + String sql = "DELETE FROM ATStatesData " + + "WHERE height BETWEEN ? AND ? " + + "AND NOT EXISTS(" + "SELECT TRUE FROM LatestATStates " + "WHERE LatestATStates.AT_address = ATStatesData.AT_address " + "AND LatestATStates.height = ATStatesData.height" - + ") " - + "LIMIT ?"; + + ") " + + "LIMIT ?"; - try { - return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight, limit); - } catch (SQLException e) { - repository.examineException(e); - throw new DataException("Unable to trim AT states in repository", e); + try { + return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight, limit); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to trim AT states in repository", e); + } } } @@ -719,57 +731,63 @@ public class HSQLDBATRepository implements ATRepository { @Override public int pruneAtStates(int minHeight, int maxHeight) throws DataException { - int deletedCount = 0; - - for (int height=minHeight; height atAddresses = new ArrayList<>(); - String updateSql = "SELECT AT_address FROM LatestATStates WHERE height = ?"; - try (ResultSet resultSet = this.repository.checkedExecute(updateSql, height)) { - if (resultSet != null) { - do { - String atAddress = resultSet.getString(1); - atAddresses.add(atAddress); - - } while (resultSet.next()); - } - } catch (SQLException e) { - throw new DataException("Unable to fetch flagged accounts from repository", e); - } - - List atStates = this.getBlockATStatesAtHeight(height); - for (ATStateData atState : atStates) { - //LOGGER.info("Found atState {} at height {}", atState.getATAddress(), atState.getHeight()); + for (int height = minHeight; height < maxHeight; height++) { // Give up if we're stopping if (Controller.isStopping()) { return deletedCount; } - if (atAddresses.contains(atState.getATAddress())) { - // We don't want to delete this AT state because it is still active - LOGGER.info("Skipping atState {} at height {}", atState.getATAddress(), atState.getHeight()); - continue; + // Get latest AT states for this height + List atAddresses = new ArrayList<>(); + String updateSql = "SELECT AT_address FROM LatestATStates WHERE height = ?"; + try (ResultSet resultSet = this.repository.checkedExecute(updateSql, height)) { + if (resultSet != null) { + do { + String atAddress = resultSet.getString(1); + atAddresses.add(atAddress); + + } while (resultSet.next()); + } + } catch (SQLException e) { + throw new DataException("Unable to fetch flagged accounts from repository", e); } - // Safe to delete everything else for this height - try { - this.repository.delete("ATStates", "AT_address = ? AND height = ?", - atState.getATAddress(), atState.getHeight()); - deletedCount++; - } catch (SQLException e) { - throw new DataException("Unable to delete AT state data from repository", e); + List atStates = this.getBlockATStatesAtHeight(height); + for (ATStateData atState : atStates) { + //LOGGER.info("Found atState {} at height {}", atState.getATAddress(), atState.getHeight()); + + // Give up if we're stopping + if (Controller.isStopping()) { + return deletedCount; + } + + if (atAddresses.contains(atState.getATAddress())) { + // We don't want to delete this AT state because it is still active + LOGGER.info("Skipping atState {} at height {}", atState.getATAddress(), atState.getHeight()); + continue; + } + + // Safe to delete everything else for this height + try { + this.repository.delete("ATStates", "AT_address = ? AND height = ?", + atState.getATAddress(), atState.getHeight()); + deletedCount++; + } catch (SQLException e) { + throw new DataException("Unable to delete AT state data from repository", e); + } } } - } - return deletedCount; + return deletedCount; + } } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 4d8e5043..3a947cd6 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -69,6 +69,7 @@ public class HSQLDBRepository implements Repository { protected final Map preparedStatementCache = new HashMap<>(); // We want the same object corresponding to the actual DB protected final Object trimHeightsLock = RepositoryManager.getRepositoryFactory(); + protected final Object latestATStatesLock = RepositoryManager.getRepositoryFactory(); private final ATRepository atRepository = new HSQLDBATRepository(this); private final AccountRepository accountRepository = new HSQLDBAccountRepository(this); From 25c17d37040120ce9ae4535c68cfd22c8c2b0478 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 3 Sep 2021 10:04:04 +0100 Subject: [PATCH 14/22] atStatesMaxLifetime reduced from 14 days to 24 hours --- src/main/java/org/qortal/settings/Settings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 84eeb3a2..6ac7342c 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -94,7 +94,7 @@ public class Settings { private int blockCacheSize = 10; /** How long to keep old, full, AT state data (ms). */ - private long atStatesMaxLifetime = 2 * 7 * 24 * 60 * 60 * 1000L; // milliseconds + private long atStatesMaxLifetime = 24 * 60 * 60 * 1000L; // milliseconds /** How often to attempt AT state trimming (ms). */ private long atStatesTrimInterval = 5678L; // milliseconds /** Block height range to scan for trimmable AT states.
From 02988989ad7f8e21f7a1524867a770c743ec3a67 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 3 Sep 2021 10:11:02 +0100 Subject: [PATCH 15/22] Reduced online account signatures min and max lifetimes onlineAccountSignaturesMinLifetime reduced from 720 hours to 12 hours onlineAccountSignaturesMaxLifetime reduced from 888 hours to 24 hours These were using up too much space in the database and so it makes sense to trim them more aggressively (assuming testing goes well). We will now stop validating online account signatures after 12 hours, which should be more than enough confirmations, and we will discard them after 24 hours. Note: this will create some complexity once some of the network is running this code. It could cause out-of-sync nodes on old versions to start treating blocks as invalid from updated peers. It's likely not worth the complexity of a hard fork though, given that almost all nodes will be synced to the chain tip and will therefore be unaffected. And even with a hard fork, we'd still face this problem on out of date nodes. --- src/main/resources/blockchain.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index d0ac9ffb..acba90da 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -11,8 +11,8 @@ "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 6, "founderEffectiveMintingLevel": 10, - "onlineAccountSignaturesMinLifetime": 2592000000, - "onlineAccountSignaturesMaxLifetime": 3196800000, + "onlineAccountSignaturesMinLifetime": 43200000, + "onlineAccountSignaturesMaxLifetime": 86400000, "rewardsByHeight": [ { "height": 1, "reward": 5.00 }, { "height": 259201, "reward": 4.75 }, From 703cdfe17458f13749285e23fd6d29b0fe81c212 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 4 Sep 2021 19:40:51 +0100 Subject: [PATCH 16/22] Added block archive mode This takes all trimmed blocks (which should now be all but the last 1450 or so) and moves them into flat files. Each file contains the serialized bytes of as many blocks that can fit within the file size target of 100MiB. As a result, the HSQLDB size drops to less than 1GB, making it much faster and easier to maintain. It also significantly reduces the total size of each full node, because the data is stored in a highly optimized way. HSQLDB then works similarly to the way it does in pruning mode - it holds all transactions, the latest state of every AT, as well as the full AT states data and hashes for the past 1450 blocks. Each archive file contains headers and indexes in order to quickly locate blocks. When a peer requests a block that is within the archive, the serialized bytes are sent directly without the need to go via a BlockData object. Now that there are no slow queries or data serialization processes needed, it should greatly speed up the block serving. The /block API endpoints have been modified in such a way that they will also check and retrieve blocks from the archive when needed. A lightweight "BlockArchive" table is needed in HSQLDB to map block heights to signatures minters and timestamps. It made more sense to keep SQL support for these basic attributes of each block. These are located in a separate table from the full blocks, in order to create a clear distinction between HSQLDB blocks and archived blocks, and also to speed up query times in the Blocks table, which is the one we are using 99% of the time. There is currently a restriction on the /admin/orphan API endpoint to prevent orphaning beyond the threshold of the block archive. --- .../qortal/api/resource/AdminResource.java | 20 ++ .../qortal/api/resource/BlocksResource.java | 270 ++++++++++++++--- .../org/qortal/controller/Controller.java | 39 ++- .../controller/repository/AtStatesPruner.java | 22 +- .../repository/AtStatesTrimmer.java | 2 +- .../controller/repository/BlockArchiver.java | 105 +++++++ .../controller/repository/BlockPruner.java | 27 +- .../controller/repository/PruneManager.java | 77 +++-- .../qortal/data/block/BlockArchiveData.java | 47 +++ .../network/message/CachedBlockMessage.java | 2 +- .../org/qortal/repository/ATRepository.java | 5 +- .../qortal/repository/BlockArchiveReader.java | 251 ++++++++++++++++ .../repository/BlockArchiveRepository.java | 118 ++++++++ .../qortal/repository/BlockArchiveWriter.java | 193 ++++++++++++ .../qortal/repository/BlockRepository.java | 5 - .../org/qortal/repository/Repository.java | 2 + .../qortal/repository/RepositoryManager.java | 21 +- .../repository/hsqldb/HSQLDBATRepository.java | 13 +- .../hsqldb/HSQLDBBlockArchiveRepository.java | 277 ++++++++++++++++++ .../hsqldb/HSQLDBBlockRepository.java | 81 +---- .../hsqldb/HSQLDBDatabaseArchiving.java | 87 ++++++ .../hsqldb/HSQLDBDatabasePruning.java | 51 +++- .../hsqldb/HSQLDBDatabaseUpdates.java | 19 ++ .../repository/hsqldb/HSQLDBRepository.java | 23 +- .../java/org/qortal/settings/Settings.java | 22 ++ 25 files changed, 1593 insertions(+), 186 deletions(-) create mode 100644 src/main/java/org/qortal/controller/repository/BlockArchiver.java create mode 100644 src/main/java/org/qortal/data/block/BlockArchiveData.java create mode 100644 src/main/java/org/qortal/repository/BlockArchiveReader.java create mode 100644 src/main/java/org/qortal/repository/BlockArchiveRepository.java create mode 100644 src/main/java/org/qortal/repository/BlockArchiveWriter.java create mode 100644 src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java create mode 100644 src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index 88dd0065..3e666fe4 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -35,6 +35,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.RollingFileAppender; import org.qortal.account.Account; @@ -67,6 +68,8 @@ import com.google.common.collect.Lists; @Tag(name = "Admin") public class AdminResource { + private static final Logger LOGGER = LogManager.getLogger(AdminResource.class); + private static final int MAX_LOG_LINES = 500; @Context @@ -459,6 +462,23 @@ public class AdminResource { if (targetHeight <= 0 || targetHeight > Controller.getInstance().getChainHeight()) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT); + // Make sure we're not orphaning as far back as the archived blocks + // FUTURE: we could support this by first importing earlier blocks from the archive + if (Settings.getInstance().isPruningEnabled() || + Settings.getInstance().isArchiveEnabled()) { + + try (final Repository repository = RepositoryManager.getRepository()) { + // Find the first unarchived block + int oldestBlock = repository.getBlockArchiveRepository().getBlockArchiveHeight(); + // Add some extra blocks just in case we're currently archiving/pruning + oldestBlock += 100; + if (targetHeight <= oldestBlock) { + LOGGER.info("Unable to orphan beyond block {} because it is archived", oldestBlock); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT); + } + } + } + if (BlockChain.orphan(targetHeight)) return "true"; else diff --git a/src/main/java/org/qortal/api/resource/BlocksResource.java b/src/main/java/org/qortal/api/resource/BlocksResource.java index 8920ecc1..6dc13c8a 100644 --- a/src/main/java/org/qortal/api/resource/BlocksResource.java +++ b/src/main/java/org/qortal/api/resource/BlocksResource.java @@ -15,6 +15,8 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.List; import javax.servlet.http.HttpServletRequest; @@ -33,11 +35,13 @@ import org.qortal.api.ApiExceptionFactory; import org.qortal.api.model.BlockMintingInfo; import org.qortal.api.model.BlockSignerSummary; import org.qortal.block.Block; +import org.qortal.controller.Controller; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountData; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.BlockArchiveReader; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -81,11 +85,19 @@ public class BlocksResource { } try (final Repository repository = RepositoryManager.getRepository()) { + // Check the database first BlockData blockData = repository.getBlockRepository().fromSignature(signature); - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + if (blockData != null) { + return blockData; + } - return blockData; + // Not found, so try the block archive + blockData = repository.getBlockArchiveRepository().fromSignature(signature); + if (blockData != null) { + return blockData; + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -116,16 +128,24 @@ public class BlocksResource { } try (final Repository repository = RepositoryManager.getRepository()) { - BlockData blockData = repository.getBlockRepository().fromSignature(signature); - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); - - Block block = new Block(repository, blockData); - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); - bytes.write(BlockTransformer.toBytes(block)); - return Base58.encode(bytes.toByteArray()); + // Check the database first + BlockData blockData = repository.getBlockRepository().fromSignature(signature); + if (blockData != null) { + Block block = new Block(repository, blockData); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); + bytes.write(BlockTransformer.toBytes(block)); + return Base58.encode(bytes.toByteArray()); + } + + // Not found, so try the block archive + byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, repository); + if (bytes != null) { + return Base58.encode(bytes); + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); } catch (TransformationException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); } catch (DataException | IOException e) { @@ -170,8 +190,12 @@ public class BlocksResource { } try (final Repository repository = RepositoryManager.getRepository()) { - if (repository.getBlockRepository().getHeightFromSignature(signature) == 0) + // Check if the block exists in either the database or archive + if (repository.getBlockRepository().getHeightFromSignature(signature) == 0 && + repository.getBlockArchiveRepository().getHeightFromSignature(signature) == 0) { + // Not found in either the database or archive throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } return repository.getBlockRepository().getTransactionsFromSignature(signature, limit, offset, reverse); } catch (DataException e) { @@ -200,7 +224,19 @@ public class BlocksResource { }) public BlockData getFirstBlock() { try (final Repository repository = RepositoryManager.getRepository()) { - return repository.getBlockRepository().fromHeight(1); + // Check the database first + BlockData blockData = repository.getBlockRepository().fromHeight(1); + if (blockData != null) { + return blockData; + } + + // Try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(1); + if (blockData != null) { + return blockData; + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -262,17 +298,28 @@ public class BlocksResource { } try (final Repository repository = RepositoryManager.getRepository()) { - BlockData blockData = repository.getBlockRepository().fromSignature(signature); + BlockData childBlockData = null; - // Check block exists - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + // Check if block exists in database + BlockData blockData = repository.getBlockRepository().fromSignature(signature); + if (blockData != null) { + return repository.getBlockRepository().fromReference(signature); + } - BlockData childBlockData = repository.getBlockRepository().fromReference(signature); + // Not found, so try the archive + // This also checks that the parent block exists + // It will return null if either the parent or child don't exit + childBlockData = repository.getBlockArchiveRepository().fromReference(signature); // Check child block exists - if (childBlockData == null) + if (childBlockData == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + + // Check child block's reference matches the supplied signature + if (!Arrays.equals(childBlockData.getReference(), signature)) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } return childBlockData; } catch (DataException e) { @@ -338,13 +385,20 @@ public class BlocksResource { } try (final Repository repository = RepositoryManager.getRepository()) { + // Firstly check the database BlockData blockData = repository.getBlockRepository().fromSignature(signature); + if (blockData != null) { + return blockData.getHeight(); + } - // Check block exists - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + // Not found, so try the archive + blockData = repository.getBlockArchiveRepository().fromSignature(signature); + if (blockData != null) { + return blockData.getHeight(); + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); - return blockData.getHeight(); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -371,11 +425,20 @@ public class BlocksResource { }) public BlockData getByHeight(@PathParam("height") int height) { try (final Repository repository = RepositoryManager.getRepository()) { + // Firstly check the database BlockData blockData = repository.getBlockRepository().fromHeight(height); - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + if (blockData != null) { + return blockData; + } + + // Not found, so try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(height); + if (blockData != null) { + return blockData; + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); - return blockData; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -402,12 +465,31 @@ public class BlocksResource { }) public BlockMintingInfo getBlockMintingInfoByHeight(@PathParam("height") int height) { try (final Repository repository = RepositoryManager.getRepository()) { + // Try the database BlockData blockData = repository.getBlockRepository().fromHeight(height); - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + if (blockData == null) { + + // Not found, so try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(height); + if (blockData == null) { + + // Still not found + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + } Block block = new Block(repository, blockData); BlockData parentBlockData = repository.getBlockRepository().fromSignature(blockData.getReference()); + if (parentBlockData == null) { + // Parent block not found - try the archive + parentBlockData = repository.getBlockArchiveRepository().fromSignature(blockData.getReference()); + if (parentBlockData == null) { + + // Still not found + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + } + int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey()); if (minterLevel == 0) // This may be unavailable when requesting a trimmed block @@ -454,13 +536,26 @@ public class BlocksResource { }) public BlockData getByTimestamp(@PathParam("timestamp") long timestamp) { try (final Repository repository = RepositoryManager.getRepository()) { + BlockData blockData = null; + + // Try the Blocks table int height = repository.getBlockRepository().getHeightFromTimestamp(timestamp); - if (height == 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + if (height > 0) { + // Found match in Blocks table + return repository.getBlockRepository().fromHeight(height); + } - BlockData blockData = repository.getBlockRepository().fromHeight(height); - if (blockData == null) + // Not found in Blocks table, so try the archive + height = repository.getBlockArchiveRepository().getHeightFromTimestamp(timestamp); + if (height > 0) { + // Found match in archive + blockData = repository.getBlockArchiveRepository().fromHeight(height); + } + + // Ensure block exists + if (blockData == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } return blockData; } catch (DataException e) { @@ -497,9 +592,14 @@ public class BlocksResource { for (/* count already set */; count > 0; --count, ++height) { BlockData blockData = repository.getBlockRepository().fromHeight(height); - if (blockData == null) - // Run out of blocks! - break; + if (blockData == null) { + // Not found - try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(height); + if (blockData == null) { + // Run out of blocks! + break; + } + } blocks.add(blockData); } @@ -544,7 +644,29 @@ public class BlocksResource { if (accountData == null || accountData.getPublicKey() == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.PUBLIC_KEY_NOT_FOUND); - return repository.getBlockRepository().getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse); + + List summaries = repository.getBlockRepository() + .getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse); + + // Add any from the archive + List archivedSummaries = repository.getBlockArchiveRepository() + .getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse); + if (archivedSummaries != null && !archivedSummaries.isEmpty()) { + summaries.addAll(archivedSummaries); + } + else { + summaries = archivedSummaries; + } + + // Sort the results (because they may have been obtained from two places) + if (reverse != null && reverse) { + summaries.sort((s1, s2) -> Integer.valueOf(s2.getHeight()).compareTo(Integer.valueOf(s1.getHeight()))); + } + else { + summaries.sort(Comparator.comparing(s -> Integer.valueOf(s.getHeight()))); + } + + return summaries; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -580,7 +702,8 @@ public class BlocksResource { if (!Crypto.isValidAddress(address)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - return repository.getBlockRepository().getBlockSigners(addresses, limit, offset, reverse); + // This method pulls data from both Blocks and BlockArchive, so no need to query serparately + return repository.getBlockArchiveRepository().getBlockSigners(addresses, limit, offset, reverse); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -620,7 +743,76 @@ public class BlocksResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); try (final Repository repository = RepositoryManager.getRepository()) { - return repository.getBlockRepository().getBlockSummaries(startHeight, endHeight, count); + + /* + * start end count result + * 10 40 null blocks 10 to 39 (excludes end block, ignore count) + * + * null null null blocks 1 to 50 (assume count=50, maybe start=1) + * 30 null null blocks 30 to 79 (assume count=50) + * 30 null 10 blocks 30 to 39 + * + * null null 50 last 50 blocks? so if max(blocks.height) is 200, then blocks 151 to 200 + * null 200 null blocks 150 to 199 (excludes end block, assume count=50) + * null 200 10 blocks 190 to 199 (excludes end block) + */ + + List blockSummaries = new ArrayList<>(); + + // Use the latest X blocks if only a count is specified + if (startHeight == null && endHeight == null && count != null) { + BlockData chainTip = Controller.getInstance().getChainTip(); + startHeight = chainTip.getHeight() - count; + endHeight = chainTip.getHeight(); + } + + // ... otherwise default the start height to 1 + if (startHeight == null && endHeight == null) { + startHeight = 1; + } + + // Default the count to 50 + if (count == null) { + count = 50; + } + + // If both a start and end height exist, ignore the count + if (startHeight != null && endHeight != null) { + if (startHeight > 0 && endHeight > 0) { + count = Integer.MAX_VALUE; + } + } + + // Derive start height from end height if missing + if (startHeight == null || startHeight == 0) { + if (endHeight != null && endHeight > 0) { + if (count != null) { + startHeight = endHeight - count; + } + } + } + + for (/* count already set */; count > 0; --count, ++startHeight) { + if (endHeight != null && startHeight >= endHeight) { + break; + } + BlockData blockData = repository.getBlockRepository().fromHeight(startHeight); + if (blockData == null) { + // Not found - try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(startHeight); + if (blockData == null) { + // Run out of blocks! + break; + } + } + + if (blockData != null) { + BlockSummaryData blockSummaryData = new BlockSummaryData(blockData); + blockSummaries.add(blockSummaryData); + } + } + + return blockSummaries; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index f9d48c70..f03dd504 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -83,20 +83,14 @@ import org.qortal.network.message.OnlineAccountsMessage; import org.qortal.network.message.SignaturesMessage; import org.qortal.network.message.TransactionMessage; import org.qortal.network.message.TransactionSignaturesMessage; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryFactory; -import org.qortal.repository.RepositoryManager; +import org.qortal.repository.*; import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.utils.Base58; -import org.qortal.utils.ByteArray; -import org.qortal.utils.NTP; -import org.qortal.utils.Triple; +import org.qortal.utils.*; import com.google.common.primitives.Longs; @@ -414,6 +408,7 @@ public class Controller extends Thread { try { RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl()); RepositoryManager.setRepositoryFactory(repositoryFactory); + RepositoryManager.archive(); RepositoryManager.prune(); } catch (DataException e) { // If exception has no cause then repository is in use by some other process. @@ -1286,6 +1281,34 @@ public class Controller extends Thread { } } + // If we have no block data, we should check the archive in case it's there + if (blockData == null) { + if (Settings.getInstance().isArchiveEnabled()) { + byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, repository); + if (bytes != null) { + CachedBlockMessage blockMessage = new CachedBlockMessage(bytes); + blockMessage.setId(message.getId()); + + // This call also causes the other needed data to be pulled in from repository + if (!peer.sendMessage(blockMessage)) { + peer.disconnect("failed to send block"); + // Don't fall-through to caching because failure to send might be from failure to build message + return; + } + + // If request is for a recent block, cache it + if (getChainHeight() - blockData.getHeight() <= blockCacheSize) { + this.stats.getBlockMessageStats.cacheFills.incrementAndGet(); + + this.blockMessageCache.put(new ByteArray(blockData.getSignature()), blockMessage); + } + + // Sent successfully from archive, so nothing more to do + return; + } + } + } + if (blockData == null) { // We don't have this block this.stats.getBlockMessageStats.unknownBlocks.getAndIncrement(); diff --git a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java index 30d7f136..1493f478 100644 --- a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java @@ -18,15 +18,24 @@ public class AtStatesPruner implements Runnable { public void run() { Thread.currentThread().setName("AT States pruner"); + boolean archiveMode = false; if (!Settings.getInstance().isPruningEnabled()) { - return; + // Pruning isn't enabled, but we might want to prune for the purposes of archiving + if (!Settings.getInstance().isArchiveEnabled()) { + // No pruning or archiving, so we must not prune anything + return; + } + else { + // We're allowed to prune blocks that have already been archived + archiveMode = true; + } } try (final Repository repository = RepositoryManager.getRepository()) { int pruneStartHeight = repository.getATRepository().getAtPruneHeight(); + repository.discardChanges(); repository.getATRepository().rebuildLatestAtStates(); - repository.saveChanges(); while (!Controller.isStopping()) { repository.discardChanges(); @@ -43,7 +52,14 @@ public class AtStatesPruner implements Runnable { // Prune AT states for all blocks up until our latest minus pruneBlockLimit final int ourLatestHeight = chainTip.getHeight(); - final int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); + int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); + + // In archive mode we are only allowed to trim blocks that have already been archived + if (archiveMode) { + upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1; + + // TODO: validate that the actual archived data exists before pruning it? + } int upperBatchHeight = pruneStartHeight + Settings.getInstance().getAtStatesPruneBatchSize(); int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); diff --git a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java index ed02ee47..98a1a889 100644 --- a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java @@ -21,8 +21,8 @@ public class AtStatesTrimmer implements Runnable { try (final Repository repository = RepositoryManager.getRepository()) { int trimStartHeight = repository.getATRepository().getAtTrimHeight(); + repository.discardChanges(); repository.getATRepository().rebuildLatestAtStates(); - repository.saveChanges(); while (!Controller.isStopping()) { repository.discardChanges(); diff --git a/src/main/java/org/qortal/controller/repository/BlockArchiver.java b/src/main/java/org/qortal/controller/repository/BlockArchiver.java new file mode 100644 index 00000000..f7bafe7d --- /dev/null +++ b/src/main/java/org/qortal/controller/repository/BlockArchiver.java @@ -0,0 +1,105 @@ +package org.qortal.controller.repository; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.data.block.BlockData; +import org.qortal.repository.*; +import org.qortal.settings.Settings; +import org.qortal.transform.TransformationException; +import org.qortal.utils.NTP; + +import java.io.IOException; + +public class BlockArchiver implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(BlockArchiver.class); + + private static final long INITIAL_SLEEP_PERIOD = 0L; // TODO: 5 * 60 * 1000L + 1234L; // ms + + public void run() { + Thread.currentThread().setName("Block archiver"); + + if (!Settings.getInstance().isArchiveEnabled()) { + return; + } + + try (final Repository repository = RepositoryManager.getRepository()) { + int startHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); + + // Don't even start building until initial rush has ended + Thread.sleep(INITIAL_SLEEP_PERIOD); + + LOGGER.info("Starting block archiver..."); + + while (!Controller.isStopping()) { + repository.discardChanges(); + + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository, true); + + Thread.sleep(Settings.getInstance().getArchiveInterval()); + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) { + continue; + } + + // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages + if (Controller.getInstance().isSynchronizing()) { + continue; + } + + // Don't attempt to archive if we're not synced yet + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) { + continue; + } + + + // Build cache of blocks + try { + BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository); + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + switch (result) { + case OK: + // Increment block archive height + startHeight += writer.getWrittenCount(); + repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight); + repository.saveChanges(); + break; + + case STOPPING: + return; + + case BLOCK_LIMIT_REACHED: + // We've reached the limit of the blocks we can archive + // Sleep for a while to allow more to become available + case NOT_ENOUGH_BLOCKS: + // We didn't reach our file size target, so that must mean that we don't have enough blocks + // yet or something went wrong. Sleep for a while and then try again. + Thread.sleep(60 * 60 * 1000L); // 1 hour + break; + + case BLOCK_NOT_FOUND: + // We tried to archive a block that didn't exist. This is a major failure and likely means + // that a bootstrap or re-sync is needed. Try again every minute until then. + LOGGER.info("Error: block not found when building archive. If this error persists, " + + "a bootstrap or re-sync may be needed."); + Thread.sleep( 60 * 1000L); // 1 minute + break; + } + + } catch (IOException | TransformationException e) { + LOGGER.info("Caught exception when creating block cache", e); + } + + } + } catch (DataException e) { + LOGGER.info("Caught exception when creating block cache", e); + } catch (InterruptedException e) { + // Do nothing + } + + } + +} diff --git a/src/main/java/org/qortal/controller/repository/BlockPruner.java b/src/main/java/org/qortal/controller/repository/BlockPruner.java index 6d3180a8..f8fd2195 100644 --- a/src/main/java/org/qortal/controller/repository/BlockPruner.java +++ b/src/main/java/org/qortal/controller/repository/BlockPruner.java @@ -18,8 +18,17 @@ public class BlockPruner implements Runnable { public void run() { Thread.currentThread().setName("Block pruner"); + boolean archiveMode = false; if (!Settings.getInstance().isPruningEnabled()) { - return; + // Pruning isn't enabled, but we might want to prune for the purposes of archiving + if (!Settings.getInstance().isArchiveEnabled()) { + // No pruning or archiving, so we must not prune anything + return; + } + else { + // We're allowed to prune blocks that have already been archived + archiveMode = true; + } } try (final Repository repository = RepositoryManager.getRepository()) { @@ -35,12 +44,24 @@ public class BlockPruner implements Runnable { continue; // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages - if (Controller.getInstance().isSynchronizing()) + if (Controller.getInstance().isSynchronizing()) { + continue; + } + + // Don't attempt to prune if we're not synced yet + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) { continue; + } // Prune all blocks up until our latest minus pruneBlockLimit final int ourLatestHeight = chainTip.getHeight(); - final int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); + int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); + + // In archive mode we are only allowed to trim blocks that have already been archived + if (archiveMode) { + upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1; + } int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize(); int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); diff --git a/src/main/java/org/qortal/controller/repository/PruneManager.java b/src/main/java/org/qortal/controller/repository/PruneManager.java index 5f92c75d..dcb21181 100644 --- a/src/main/java/org/qortal/controller/repository/PruneManager.java +++ b/src/main/java/org/qortal/controller/repository/PruneManager.java @@ -35,29 +35,70 @@ public class PruneManager { public void start() { this.executorService = Executors.newCachedThreadPool(new DaemonThreadFactory()); - // Don't allow both the pruner and the trimmer to run at the same time. - // In pruning mode, we are already deleting far more than we would when trimming. - // In non-pruning mode, we still need to trim to keep the non-essential data - // out of the database. There isn't a case where both are needed at once. - // If we ever do need to enable both at once, be very careful with the AT state - // trimming, since both currently rely on having exclusive access to the - // prepareForAtStateTrimming() method. For both trimming and pruning to take place - // at once, we would need to synchronize this method in a way that both can't - // call it at the same time, as otherwise active ATs would be pruned/trimmed when - // they should have been kept. - - if (Settings.getInstance().isPruningEnabled()) { - // Pruning enabled - start the pruning processes - this.executorService.execute(new AtStatesPruner()); - this.executorService.execute(new BlockPruner()); + if (Settings.getInstance().isPruningEnabled() && + !Settings.getInstance().isArchiveEnabled()) { + // Top-only-sync + this.startTopOnlySyncMode(); + } + else if (Settings.getInstance().isArchiveEnabled()) { + // Full node with block archive + this.startFullNodeWithBlockArchive(); } else { - // Pruning disabled - use trimming instead - this.executorService.execute(new AtStatesTrimmer()); - this.executorService.execute(new OnlineAccountsSignaturesTrimmer()); + // Full node with full SQL support + this.startFullSQLNode(); } } + /** + * Top-only-sync + * In this mode, we delete (prune) all blocks except + * a small number of recent ones. There is no need for + * trimming or archiving, because all relevant blocks + * are deleted. + */ + private void startTopOnlySyncMode() { + this.startPruning(); + } + + /** + * Full node with block archive + * In this mode we archive trimmed blocks, and then + * prune archived blocks to keep the database small + */ + private void startFullNodeWithBlockArchive() { + this.startTrimming(); + this.startArchiving(); + this.startPruning(); + } + + /** + * Full node with full SQL support + * In this mode we trim the database but don't prune + * or archive any data, because we want to maintain + * full SQL support of old blocks. This mode will not + * be actively maintained but can be used by those who + * need to perform SQL analysis on older blocks. + */ + private void startFullSQLNode() { + this.startTrimming(); + } + + + private void startPruning() { + this.executorService.execute(new AtStatesPruner()); + this.executorService.execute(new BlockPruner()); + } + + private void startTrimming() { + this.executorService.execute(new AtStatesTrimmer()); + this.executorService.execute(new OnlineAccountsSignaturesTrimmer()); + } + + private void startArchiving() { + this.executorService.execute(new BlockArchiver()); + } + public void stop() { this.executorService.shutdownNow(); diff --git a/src/main/java/org/qortal/data/block/BlockArchiveData.java b/src/main/java/org/qortal/data/block/BlockArchiveData.java new file mode 100644 index 00000000..c9db4032 --- /dev/null +++ b/src/main/java/org/qortal/data/block/BlockArchiveData.java @@ -0,0 +1,47 @@ +package org.qortal.data.block; + +import org.qortal.block.Block; + +public class BlockArchiveData { + + // Properties + private byte[] signature; + private Integer height; + private Long timestamp; + private byte[] minterPublicKey; + + // Constructors + + public BlockArchiveData(byte[] signature, Integer height, long timestamp, byte[] minterPublicKey) { + this.signature = signature; + this.height = height; + this.timestamp = timestamp; + this.minterPublicKey = minterPublicKey; + } + + public BlockArchiveData(BlockData blockData) { + this.signature = blockData.getSignature(); + this.height = blockData.getHeight(); + this.timestamp = blockData.getTimestamp(); + this.minterPublicKey = blockData.getMinterPublicKey(); + } + + // Getters/setters + + public byte[] getSignature() { + return this.signature; + } + + public Integer getHeight() { + return this.height; + } + + public Long getTimestamp() { + return this.timestamp; + } + + public byte[] getMinterPublicKey() { + return this.minterPublicKey; + } + +} diff --git a/src/main/java/org/qortal/network/message/CachedBlockMessage.java b/src/main/java/org/qortal/network/message/CachedBlockMessage.java index 7a175810..e5029ab0 100644 --- a/src/main/java/org/qortal/network/message/CachedBlockMessage.java +++ b/src/main/java/org/qortal/network/message/CachedBlockMessage.java @@ -23,7 +23,7 @@ public class CachedBlockMessage extends Message { this.block = block; } - private CachedBlockMessage(byte[] cachedBytes) { + public CachedBlockMessage(byte[] cachedBytes) { super(MessageType.BLOCK); this.block = null; diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 74fb19ab..9316875d 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -113,7 +113,10 @@ public interface ATRepository { public List getBlockATStatesAtHeight(int height) throws DataException; - /** Rebuild the latest AT states cache, necessary for AT state trimming/pruning. */ + /** Rebuild the latest AT states cache, necessary for AT state trimming/pruning. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ public void rebuildLatestAtStates() throws DataException; diff --git a/src/main/java/org/qortal/repository/BlockArchiveReader.java b/src/main/java/org/qortal/repository/BlockArchiveReader.java new file mode 100644 index 00000000..1b68a7c5 --- /dev/null +++ b/src/main/java/org/qortal/repository/BlockArchiveReader.java @@ -0,0 +1,251 @@ +package org.qortal.repository; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockArchiveData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.settings.Settings; +import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformer; +import org.qortal.utils.Triple; + +import static org.qortal.transform.Transformer.INT_LENGTH; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + +public class BlockArchiveReader { + + private static BlockArchiveReader instance; + private Map> fileListCache = Collections.synchronizedMap(new HashMap<>()); + + private static final Logger LOGGER = LogManager.getLogger(BlockArchiveReader.class); + + public BlockArchiveReader() { + + } + + public static synchronized BlockArchiveReader getInstance() { + if (instance == null) { + instance = new BlockArchiveReader(); + } + + return instance; + } + + private void fetchFileList() { + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath(); + File archiveDirFile = archivePath.toFile(); + String[] files = archiveDirFile.list(); + Map> map = new HashMap<>(); + + for (String file : files) { + Path filePath = Paths.get(file); + String filename = filePath.getFileName().toString(); + + // Parse the filename + if (filename == null || !filename.contains("-") || !filename.contains(".")) { + // Not a usable file + continue; + } + // Remove the extension and split into two parts + String[] parts = filename.substring(0, filename.lastIndexOf('.')).split("-"); + Integer startHeight = Integer.parseInt(parts[0]); + Integer endHeight = Integer.parseInt(parts[1]); + Integer range = endHeight - startHeight; + map.put(filename, new Triple(startHeight, endHeight, range)); + } + this.fileListCache = map; + } + + public Triple, List> fetchBlockAtHeight(int height) { + if (this.fileListCache.isEmpty()) { + this.fetchFileList(); + } + + byte[] serializedBytes = this.fetchSerializedBlockBytesForHeight(height); + if (serializedBytes == null) { + return null; + } + + ByteBuffer byteBuffer = ByteBuffer.wrap(serializedBytes); + Triple, List> blockInfo = null; + try { + blockInfo = BlockTransformer.fromByteBuffer(byteBuffer); + if (blockInfo != null && blockInfo.getA() != null) { + // Block height is stored outside of the main serialized bytes, so it + // won't be set automatically. + blockInfo.getA().setHeight(height); + } + } catch (TransformationException e) { + return null; + } + return blockInfo; + } + + public Triple, List> fetchBlockWithSignature( + byte[] signature, Repository repository) { + + if (this.fileListCache.isEmpty()) { + this.fetchFileList(); + } + + Integer height = this.fetchHeightForSignature(signature, repository); + if (height != null) { + return this.fetchBlockAtHeight(height); + } + return null; + } + + public Integer fetchHeightForSignature(byte[] signature, Repository repository) { + // Lookup the height for the requested signature + try { + BlockArchiveData archivedBlock = repository.getBlockArchiveRepository().getBlockArchiveDataForSignature(signature); + if (archivedBlock.getHeight() == null) { + return null; + } + return archivedBlock.getHeight(); + + } catch (DataException e) { + return null; + } + } + + public int fetchHeightForTimestamp(long timestamp, Repository repository) { + // Lookup the height for the requested signature + try { + return repository.getBlockArchiveRepository().getHeightFromTimestamp(timestamp); + + } catch (DataException e) { + return 0; + } + } + + private String getFilenameForHeight(int height) { + Iterator it = this.fileListCache.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry pair = (Map.Entry)it.next(); + if (pair == null && pair.getKey() == null && pair.getValue() == null) { + continue; + } + Triple heightInfo = (Triple) pair.getValue(); + Integer startHeight = heightInfo.getA(); + Integer endHeight = heightInfo.getB(); + + if (height >= startHeight && height <= endHeight) { + // Found the correct file + String filename = (String) pair.getKey(); + return filename; + } + } + + return null; + } + + public byte[] fetchSerializedBlockBytesForSignature(byte[] signature, Repository repository) { + + if (this.fileListCache.isEmpty()) { + this.fetchFileList(); + } + + Integer height = this.fetchHeightForSignature(signature, repository); + if (height != null) { + return this.fetchSerializedBlockBytesForHeight(height); + } + return null; + } + + public byte[] fetchSerializedBlockBytesForHeight(int height) { + String filename = this.getFilenameForHeight(height); + if (filename == null) { + // We don't have this block in the archive + // Invalidate the file list cache in case it is out of date + this.invalidateFileListCache(); + return null; + } + + Path filePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive", filename).toAbsolutePath(); + RandomAccessFile file = null; + try { + file = new RandomAccessFile(filePath.toString(), "r"); + // Get info about this file (the "fixed length header") + final int version = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + final int startHeight = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + final int endHeight = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + file.readInt(); // Block count (unused) // Do not remove or comment out, as it is moving the file pointer + final int variableHeaderLength = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + final int fixedHeaderLength = (int)file.getFilePointer(); + // End of fixed length header + + // Make sure the version is one we recognize + if (version != 1) { + LOGGER.info("Error: unknown version in file {}: {}", filename, version); + return null; + } + + // Verify that the block is within the reported range + if (height < startHeight || height > endHeight) { + LOGGER.info("Error: requested height {} but the range of file {} is {}-{}", + height, filename, startHeight, endHeight); + return null; + } + + // Seek to the location of the block index in the variable length header + final int locationOfBlockIndexInVariableHeaderSegment = (height - startHeight) * INT_LENGTH; + file.seek(fixedHeaderLength + locationOfBlockIndexInVariableHeaderSegment); + + // Read the value to obtain the index of this block in the data segment + int locationOfBlockInDataSegment = file.readInt(); + + // Now seek to the block data itself + int dataSegmentStartIndex = fixedHeaderLength + variableHeaderLength + INT_LENGTH; // Confirmed correct + file.seek(dataSegmentStartIndex + locationOfBlockInDataSegment); + + // Read the block metadata + int blockHeight = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + int blockLength = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + + // Ensure the block height matches the one requested + if (blockHeight != height) { + LOGGER.info("Error: height {} does not match requested: {}", blockHeight, height); + return null; + } + + // Now retrieve the block's serialized bytes + byte[] blockBytes = new byte[blockLength]; + file.read(blockBytes); + + return blockBytes; + + } catch (FileNotFoundException e) { + LOGGER.info("File {} not found: {}", filename, e.getMessage()); + return null; + } catch (IOException e) { + LOGGER.info("Unable to read block {} from archive: {}", height, e.getMessage()); + return null; + } + finally { + // Close the file + if (file != null) { + try { + file.close(); + } catch (IOException e) { + // Failed to close, but no need to handle this + } + } + } + } + + public void invalidateFileListCache() { + this.fileListCache.clear(); + } + +} diff --git a/src/main/java/org/qortal/repository/BlockArchiveRepository.java b/src/main/java/org/qortal/repository/BlockArchiveRepository.java new file mode 100644 index 00000000..c702a7ef --- /dev/null +++ b/src/main/java/org/qortal/repository/BlockArchiveRepository.java @@ -0,0 +1,118 @@ +package org.qortal.repository; + +import org.qortal.api.model.BlockSignerSummary; +import org.qortal.data.block.BlockArchiveData; +import org.qortal.data.block.BlockData; +import org.qortal.data.block.BlockSummaryData; + +import java.util.List; + +public interface BlockArchiveRepository { + + /** + * Returns BlockData from archive using block signature. + * + * @param signature + * @return block data, or null if not found in archive. + * @throws DataException + */ + public BlockData fromSignature(byte[] signature) throws DataException; + + /** + * Return height of block in archive using block's signature. + * + * @param signature + * @return height, or 0 if not found in blockchain. + * @throws DataException + */ + public int getHeightFromSignature(byte[] signature) throws DataException; + + /** + * Returns BlockData from archive using block height. + * + * @param height + * @return block data, or null if not found in blockchain. + * @throws DataException + */ + public BlockData fromHeight(int height) throws DataException; + + /** + * Returns BlockData from archive using block reference. + * Currently relies on a child block being the one block + * higher than its parent. This limitation can be removed + * by storing the reference in the BlockArchive table, but + * this has been avoided to reduce space. + * + * @param reference + * @return block data, or null if either parent or child + * not found in the archive. + * @throws DataException + */ + public BlockData fromReference(byte[] reference) throws DataException; + + /** + * Return height of block with timestamp just before passed timestamp. + * + * @param timestamp + * @return height, or 0 if not found in blockchain. + * @throws DataException + */ + public int getHeightFromTimestamp(long timestamp) throws DataException; + + /** + * Returns block summaries for blocks signed by passed public key, or reward-share with minter with passed public key. + */ + public List getBlockSummariesBySigner(byte[] signerPublicKey, Integer limit, Integer offset, Boolean reverse) throws DataException; + + /** + * Returns summaries of block signers, optionally limited to passed addresses. + * This combines both the BlockArchive and the Blocks data into a single result set. + */ + public List getBlockSigners(List addresses, Integer limit, Integer offset, Boolean reverse) throws DataException; + + + /** Returns height of first unarchived block. */ + public int getBlockArchiveHeight() throws DataException; + + /** Sets new height for block archiving. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ + public void setBlockArchiveHeight(int archiveHeight) throws DataException; + + + /** + * Returns the block archive data for a given signature, from the block archive. + *

+ * This method will return null if no block archive has been built for the + * requested signature. In those cases, the height (and other data) can be + * looked up using the Blocks table. This allows a block to be located in + * the archive when we only know its signature. + *

+ * + * @param signature + * @throws DataException + */ + public BlockArchiveData getBlockArchiveDataForSignature(byte[] signature) throws DataException; + + /** + * Saves a block archive entry into the repository. + *

+ * This can be used to find the height of a block by its signature, without + * having access to the block data itself. + *

+ * + * @param blockArchiveData + * @throws DataException + */ + public void save(BlockArchiveData blockArchiveData) throws DataException; + + /** + * Deletes a block archive entry from the repository. + * + * @param blockArchiveData + * @throws DataException + */ + public void delete(BlockArchiveData blockArchiveData) throws DataException; + +} diff --git a/src/main/java/org/qortal/repository/BlockArchiveWriter.java b/src/main/java/org/qortal/repository/BlockArchiveWriter.java new file mode 100644 index 00000000..4aeb1a32 --- /dev/null +++ b/src/main/java/org/qortal/repository/BlockArchiveWriter.java @@ -0,0 +1,193 @@ +package org.qortal.repository; + +import com.google.common.primitives.Ints; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.block.Block; +import org.qortal.controller.Controller; +import org.qortal.data.block.BlockArchiveData; +import org.qortal.data.block.BlockData; +import org.qortal.settings.Settings; +import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformer; + +import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class BlockArchiveWriter { + + public enum BlockArchiveWriteResult { + OK, + STOPPING, + NOT_ENOUGH_BLOCKS, + BLOCK_LIMIT_REACHED, + BLOCK_NOT_FOUND + } + + private static final Logger LOGGER = LogManager.getLogger(BlockArchiveWriter.class); + + private int startHeight; + private final int endHeight; + private final Repository repository; + + private int writtenCount; + + public BlockArchiveWriter(int startHeight, int endHeight, Repository repository) { + this.startHeight = startHeight; + this.endHeight = endHeight; + this.repository = repository; + } + + public static int getMaxArchiveHeight(Repository repository, boolean useMaximumDuplicatedLimit) throws DataException { + // We must only archive trimmed blocks, or the archive will grow far too large + final int accountSignaturesTrimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight(); + final int atTrimStartHeight = repository.getATRepository().getAtTrimHeight(); + final int trimStartHeight = Math.min(accountSignaturesTrimStartHeight, atTrimStartHeight); + + // In some cases we want to restrict the upper height of the archiver to save space + if (useMaximumDuplicatedLimit) { + // To save on disk space, it's best to not allow the archiver to get too far ahead of the pruner + // This reduces the amount of data that is held twice during the transition + final int blockPruneStartHeight = repository.getBlockRepository().getBlockPruneHeight(); + final int atPruneStartHeight = repository.getATRepository().getAtPruneHeight(); + final int pruneStartHeight = Math.min(blockPruneStartHeight, atPruneStartHeight); + final int maximumDuplicatedBlocks = Settings.getInstance().getMaxDuplicatedBlocksWhenArchiving(); + + // To summarize the above: + // - We must never archive anything greater than or equal to trimStartHeight + // - We should avoid archiving anything maximumDuplicatedBlocks higher than pruneStartHeight + return Math.min(trimStartHeight, pruneStartHeight + maximumDuplicatedBlocks); + } + else { + // We don't want to apply the maximum duplicated limit + return trimStartHeight; + } + } + + public static boolean isArchiverUpToDate(Repository repository, boolean useMaximumDuplicatedLimit) throws DataException { + final int maxArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository, useMaximumDuplicatedLimit); + final int actualArchiveHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); + final float progress = (float)actualArchiveHeight / (float) maxArchiveHeight; + LOGGER.info(String.format("maxArchiveHeight: %d, actualArchiveHeight: %d, progress: %f", + maxArchiveHeight, actualArchiveHeight, progress)); + + // If archiver is within 90% of the maximum, treat it as up to date + // We need several percent as an allowance because the archiver will only + // save files when they reach the target size + return (progress >= 0.90); + } + + public BlockArchiveWriteResult write() throws DataException, IOException, TransformationException, InterruptedException { + // Create the archive folder if it doesn't exist + // This is a subfolder of the db directory, to make bootstrapping easier + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath(); + try { + Files.createDirectories(archivePath); + } catch (IOException e) { + LOGGER.info("Unable to create archive folder"); + throw new DataException("Unable to create archive folder"); + } + + // Determine start height of blocks to fetch + if (startHeight <= 2) { + // Skip genesis block, as it's not designed to be transmitted, and we can build that from blockchain.json + // TODO: include genesis block if we can + startHeight = 2; + } + + // Header bytes will store the block indexes + ByteArrayOutputStream headerBytes = new ByteArrayOutputStream(); + // Bytes will store the actual block data + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + LOGGER.info(String.format("Fetching blocks from height %d...", startHeight)); + int i = 0; + long fileSizeTarget = 100 * 1024 * 1024; // 100MiB + while (headerBytes.size() + bytes.size() < fileSizeTarget) { + if (Controller.isStopping()) { + return BlockArchiveWriteResult.STOPPING; + } + if (Controller.getInstance().isSynchronizing()) { + continue; + } + + int currentHeight = startHeight + i; + if (currentHeight >= endHeight) { + return BlockArchiveWriteResult.BLOCK_LIMIT_REACHED; + } + + //LOGGER.info("Fetching block {}...", currentHeight); + + BlockData blockData = repository.getBlockRepository().fromHeight(currentHeight); + if (blockData == null) { + return BlockArchiveWriteResult.BLOCK_NOT_FOUND; + } + + // Write the signature and height into the BlockArchive table + BlockArchiveData blockArchiveData = new BlockArchiveData(blockData); + repository.getBlockArchiveRepository().save(blockArchiveData); + repository.saveChanges(); + + // Write the block data to some byte buffers + Block block = new Block(repository, blockData); + int blockIndex = bytes.size(); + // Write block index to header + headerBytes.write(Ints.toByteArray(blockIndex)); + // Write block height + bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); + byte[] blockBytes = BlockTransformer.toBytes(block); + // Write block length + bytes.write(Ints.toByteArray(blockBytes.length)); + // Write block bytes + bytes.write(blockBytes); + i++; + + } + int totalLength = headerBytes.size() + bytes.size(); + LOGGER.info(String.format("Total length of %d blocks is %d bytes", i, totalLength)); + + // Validate file size, in case something went wrong + if (totalLength < fileSizeTarget) { + return BlockArchiveWriteResult.NOT_ENOUGH_BLOCKS; + } + + // We have enough blocks to create a new file + int endHeight = startHeight + i - 1; + int version = 1; + String filePath = String.format("%s/%d-%d.dat", archivePath.toString(), startHeight, endHeight); + FileOutputStream fileOutputStream = new FileOutputStream(filePath); + // Write version number + fileOutputStream.write(Ints.toByteArray(version)); + // Write start height + fileOutputStream.write(Ints.toByteArray(startHeight)); + // Write end height + fileOutputStream.write(Ints.toByteArray(endHeight)); + // Write total count + fileOutputStream.write(Ints.toByteArray(i)); + // Write dynamic header (block indexes) segment length + fileOutputStream.write(Ints.toByteArray(headerBytes.size())); + // Write dynamic header (block indexes) data + headerBytes.writeTo(fileOutputStream); + // Write data segment (block data) length + fileOutputStream.write(Ints.toByteArray(bytes.size())); + // Write data + bytes.writeTo(fileOutputStream); + // Close the file + fileOutputStream.close(); + + // Invalidate cache so that the rest of the app picks up the new file + BlockArchiveReader.getInstance().invalidateFileListCache(); + + this.writtenCount = i; + return BlockArchiveWriteResult.OK; + } + + public int getWrittenCount() { + return this.writtenCount; + } + +} diff --git a/src/main/java/org/qortal/repository/BlockRepository.java b/src/main/java/org/qortal/repository/BlockRepository.java index 5ca61e66..76891c36 100644 --- a/src/main/java/org/qortal/repository/BlockRepository.java +++ b/src/main/java/org/qortal/repository/BlockRepository.java @@ -137,11 +137,6 @@ public interface BlockRepository { */ public List getBlockSummaries(int firstBlockHeight, int lastBlockHeight) throws DataException; - /** - * Returns block summaries for the passed height range, for API use. - */ - public List getBlockSummaries(Integer startHeight, Integer endHeight, Integer count) throws DataException; - /** Returns height of first trimmable online accounts signatures. */ public int getOnlineAccountsSignaturesTrimHeight() throws DataException; diff --git a/src/main/java/org/qortal/repository/Repository.java b/src/main/java/org/qortal/repository/Repository.java index 656e6e1e..fab48a14 100644 --- a/src/main/java/org/qortal/repository/Repository.java +++ b/src/main/java/org/qortal/repository/Repository.java @@ -12,6 +12,8 @@ public interface Repository extends AutoCloseable { public BlockRepository getBlockRepository(); + public BlockArchiveRepository getBlockArchiveRepository(); + public ChatRepository getChatRepository(); public CrossChainRepository getCrossChainRepository(); diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index 5e9c71c2..f7557750 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -2,6 +2,7 @@ package org.qortal.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.repository.hsqldb.HSQLDBDatabaseArchiving; import org.qortal.repository.hsqldb.HSQLDBDatabasePruning; import org.qortal.settings.Settings; @@ -57,9 +58,23 @@ public abstract class RepositoryManager { } } - public static void prune() { + public static boolean archive() { + // Bulk archive the database the first time we use archive mode + if (Settings.getInstance().isArchiveEnabled()) { + try { + return HSQLDBDatabaseArchiving.buildBlockArchive(); + + } catch (DataException e) { + LOGGER.info("Unable to bulk prune AT states. The database may have been left in an inconsistent state."); + } + } + return false; + } + + public static boolean prune() { // Bulk prune the database the first time we use pruning mode - if (Settings.getInstance().isPruningEnabled()) { + if (Settings.getInstance().isPruningEnabled() || + Settings.getInstance().isArchiveEnabled()) { try { boolean prunedATStates = HSQLDBDatabasePruning.pruneATStates(); boolean prunedBlocks = HSQLDBDatabasePruning.pruneBlocks(); @@ -67,12 +82,14 @@ public abstract class RepositoryManager { // Perform repository maintenance to shrink the db size down if (prunedATStates && prunedBlocks) { HSQLDBDatabasePruning.performMaintenance(); + return true; } } catch (SQLException | DataException e) { LOGGER.info("Unable to bulk prune AT states. The database may have been left in an inconsistent state."); } } + return false; } public static void setRequestedCheckpoint(Boolean quick) { diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index 522fafb7..e0baa136 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -608,6 +608,7 @@ public class HSQLDBATRepository implements ATRepository { // that could result in one process using a partial or empty dataset // because it was in the process of being rebuilt by another thread synchronized (this.repository.latestATStatesLock) { + LOGGER.trace("Rebuilding latest AT states..."); // Rebuild cache of latest AT states that we can't trim String deleteSql = "DELETE FROM LatestATStates"; @@ -632,6 +633,8 @@ public class HSQLDBATRepository implements ATRepository { repository.examineException(e); throw new DataException("Unable to populate temporary latest AT states cache in repository", e); } + this.repository.saveChanges(); + LOGGER.trace("Rebuilt latest AT states"); } } @@ -661,7 +664,7 @@ public class HSQLDBATRepository implements ATRepository { this.repository.executeCheckedUpdate(updateSql, trimHeight); this.repository.saveChanges(); } catch (SQLException e) { - repository.examineException(e); + this.repository.examineException(e); throw new DataException("Unable to set AT state trim height in repository", e); } } @@ -689,7 +692,10 @@ public class HSQLDBATRepository implements ATRepository { + "LIMIT ?"; try { - return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight, limit); + int modifiedRows = this.repository.executeCheckedUpdate(sql, minHeight, maxHeight, limit); + this.repository.saveChanges(); + return modifiedRows; + } catch (SQLException e) { repository.examineException(e); throw new DataException("Unable to trim AT states in repository", e); @@ -757,7 +763,7 @@ public class HSQLDBATRepository implements ATRepository { } while (resultSet.next()); } } catch (SQLException e) { - throw new DataException("Unable to fetch flagged accounts from repository", e); + throw new DataException("Unable to fetch latest AT states from repository", e); } List atStates = this.getBlockATStatesAtHeight(height); @@ -785,6 +791,7 @@ public class HSQLDBATRepository implements ATRepository { } } } + this.repository.saveChanges(); return deletedCount; } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java new file mode 100644 index 00000000..c491f862 --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java @@ -0,0 +1,277 @@ +package org.qortal.repository.hsqldb; + +import org.qortal.api.ApiError; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.model.BlockSignerSummary; +import org.qortal.data.block.BlockArchiveData; +import org.qortal.data.block.BlockData; +import org.qortal.data.block.BlockSummaryData; +import org.qortal.repository.BlockArchiveReader; +import org.qortal.repository.BlockArchiveRepository; +import org.qortal.repository.DataException; +import org.qortal.utils.Triple; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository { + + protected HSQLDBRepository repository; + + public HSQLDBBlockArchiveRepository(HSQLDBRepository repository) { + this.repository = repository; + } + + + @Override + public BlockData fromSignature(byte[] signature) throws DataException { + Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockWithSignature(signature, this.repository); + if (blockInfo != null) { + return (BlockData) blockInfo.getA(); + } + return null; + } + + @Override + public int getHeightFromSignature(byte[] signature) throws DataException { + Integer height = BlockArchiveReader.getInstance().fetchHeightForSignature(signature, this.repository); + if (height == null || height == 0) { + return 0; + } + return height; + } + + @Override + public BlockData fromHeight(int height) throws DataException { + Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height); + if (blockInfo != null) { + return (BlockData) blockInfo.getA(); + } + return null; + } + + @Override + public BlockData fromReference(byte[] reference) throws DataException { + BlockData referenceBlock = this.repository.getBlockArchiveRepository().fromSignature(reference); + if (referenceBlock != null) { + int height = referenceBlock.getHeight(); + if (height > 0) { + // Request the block at height + 1 + Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height + 1); + if (blockInfo != null) { + return (BlockData) blockInfo.getA(); + } + } + } + return null; + } + + @Override + public int getHeightFromTimestamp(long timestamp) throws DataException { + String sql = "SELECT height FROM BlockArchive WHERE minted_when <= ? ORDER BY minted_when DESC, height DESC LIMIT 1"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, timestamp)) { + if (resultSet == null) { + return 0; + } + return resultSet.getInt(1); + + } catch (SQLException e) { + throw new DataException("Error fetching height from BlockArchive repository", e); + } + } + + @Override + public List getBlockSummariesBySigner(byte[] signerPublicKey, Integer limit, Integer offset, Boolean reverse) throws DataException { + StringBuilder sql = new StringBuilder(512); + sql.append("SELECT signature, height, BlockArchive.minter FROM "); + + // List of minter account's public key and reward-share public keys with minter's public key + sql.append("(SELECT * FROM (VALUES (CAST(? AS QortalPublicKey))) UNION (SELECT reward_share_public_key FROM RewardShares WHERE minter_public_key = ?)) AS PublicKeys (public_key) "); + + // Match BlockArchive blocks signed with public key from above list + sql.append("JOIN BlockArchive ON BlockArchive.minter = public_key "); + + sql.append("ORDER BY BlockArchive.height "); + if (reverse != null && reverse) + sql.append("DESC "); + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List blockSummaries = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), signerPublicKey, signerPublicKey)) { + if (resultSet == null) + return blockSummaries; + + do { + byte[] signature = resultSet.getBytes(1); + int height = resultSet.getInt(2); + byte[] blockMinterPublicKey = resultSet.getBytes(3); + + // Fetch additional info from the archive itself + int onlineAccountsCount = 0; + BlockData blockData = this.fromSignature(signature); + if (blockData != null) { + onlineAccountsCount = blockData.getOnlineAccountsCount(); + } + + BlockSummaryData blockSummary = new BlockSummaryData(height, signature, blockMinterPublicKey, onlineAccountsCount); + blockSummaries.add(blockSummary); + } while (resultSet.next()); + + return blockSummaries; + } catch (SQLException e) { + throw new DataException("Unable to fetch minter's block summaries from repository", e); + } + } + + @Override + public List getBlockSigners(List addresses, Integer limit, Integer offset, Boolean reverse) throws DataException { + String subquerySql = "SELECT minter, COUNT(signature) FROM (" + + "(SELECT minter, signature FROM Blocks) UNION ALL (SELECT minter, signature FROM BlockArchive)" + + ") GROUP BY minter"; + + StringBuilder sql = new StringBuilder(1024); + sql.append("SELECT DISTINCT block_minter, n_blocks, minter_public_key, minter, recipient FROM ("); + sql.append(subquerySql); + sql.append(") AS Minters (block_minter, n_blocks) LEFT OUTER JOIN RewardShares ON reward_share_public_key = block_minter "); + + if (addresses != null && !addresses.isEmpty()) { + sql.append(" LEFT OUTER JOIN Accounts AS BlockMinterAccounts ON BlockMinterAccounts.public_key = block_minter "); + sql.append(" LEFT OUTER JOIN Accounts AS RewardShareMinterAccounts ON RewardShareMinterAccounts.public_key = minter_public_key "); + sql.append(" JOIN (VALUES "); + + final int addressesSize = addresses.size(); + for (int ai = 0; ai < addressesSize; ++ai) { + if (ai != 0) + sql.append(", "); + + sql.append("(?)"); + } + + sql.append(") AS FilterAccounts (account) "); + sql.append(" ON FilterAccounts.account IN (recipient, BlockMinterAccounts.account, RewardShareMinterAccounts.account) "); + } else { + addresses = Collections.emptyList(); + } + + sql.append("ORDER BY n_blocks "); + if (reverse != null && reverse) + sql.append("DESC "); + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List summaries = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), addresses.toArray())) { + if (resultSet == null) + return summaries; + + do { + byte[] blockMinterPublicKey = resultSet.getBytes(1); + int nBlocks = resultSet.getInt(2); + + // May not be present if no reward-share: + byte[] mintingAccountPublicKey = resultSet.getBytes(3); + String minterAccount = resultSet.getString(4); + String recipientAccount = resultSet.getString(5); + + BlockSignerSummary blockSignerSummary; + if (recipientAccount == null) + blockSignerSummary = new BlockSignerSummary(blockMinterPublicKey, nBlocks); + else + blockSignerSummary = new BlockSignerSummary(blockMinterPublicKey, nBlocks, mintingAccountPublicKey, minterAccount, recipientAccount); + + summaries.add(blockSignerSummary); + } while (resultSet.next()); + + return summaries; + } catch (SQLException e) { + throw new DataException("Unable to fetch block minters from repository", e); + } + } + + + @Override + public int getBlockArchiveHeight() throws DataException { + String sql = "SELECT block_archive_height FROM DatabaseInfo"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return 0; + + return resultSet.getInt(1); + } catch (SQLException e) { + throw new DataException("Unable to fetch block archive height from repository", e); + } + } + + @Override + public void setBlockArchiveHeight(int archiveHeight) throws DataException { + // trimHeightsLock is to prevent concurrent update on DatabaseInfo + // that could result in "transaction rollback: serialization failure" + synchronized (this.repository.trimHeightsLock) { + String updateSql = "UPDATE DatabaseInfo SET block_archive_height = ?"; + + try { + this.repository.executeCheckedUpdate(updateSql, archiveHeight); + this.repository.saveChanges(); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to set block archive height in repository", e); + } + } + } + + + @Override + public BlockArchiveData getBlockArchiveDataForSignature(byte[] signature) throws DataException { + String sql = "SELECT height, signature, minted_when, minter FROM BlockArchive WHERE signature = ? LIMIT 1"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, signature)) { + if (resultSet == null) { + return null; + } + int height = resultSet.getInt(1); + byte[] sig = resultSet.getBytes(2); + long timestamp = resultSet.getLong(3); + byte[] minterPublicKey = resultSet.getBytes(4); + return new BlockArchiveData(sig, height, timestamp, minterPublicKey); + + } catch (SQLException e) { + throw new DataException("Error fetching height from BlockArchive repository", e); + } + } + + + @Override + public void save(BlockArchiveData blockArchiveData) throws DataException { + HSQLDBSaver saveHelper = new HSQLDBSaver("BlockArchive"); + + saveHelper.bind("signature", blockArchiveData.getSignature()) + .bind("height", blockArchiveData.getHeight()) + .bind("minted_when", blockArchiveData.getTimestamp()) + .bind("minter", blockArchiveData.getMinterPublicKey()); + + try { + saveHelper.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save SimpleBlockData into BlockArchive repository", e); + } + } + + @Override + public void delete(BlockArchiveData blockArchiveData) throws DataException { + try { + this.repository.delete("BlockArchive", + "block_signature = ?", blockArchiveData.getSignature()); + } catch (SQLException e) { + throw new DataException("Unable to delete SimpleBlockData from BlockArchive repository", e); + } + } + +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java index 2f7e4ad2..b8238085 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java @@ -10,6 +10,7 @@ import org.qortal.api.model.BlockSignerSummary; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.block.BlockTransactionData; +import org.qortal.data.block.BlockArchiveData; import org.qortal.data.transaction.TransactionData; import org.qortal.repository.BlockRepository; import org.qortal.repository.DataException; @@ -382,86 +383,6 @@ public class HSQLDBBlockRepository implements BlockRepository { } } - @Override - public List getBlockSummaries(Integer startHeight, Integer endHeight, Integer count) throws DataException { - StringBuilder sql = new StringBuilder(512); - List bindParams = new ArrayList<>(); - - sql.append("SELECT signature, height, minter, online_accounts_count, minted_when, transaction_count "); - - /* - * start end count result - * 10 40 null blocks 10 to 39 (excludes end block, ignore count) - * - * null null null blocks 1 to 50 (assume count=50, maybe start=1) - * 30 null null blocks 30 to 79 (assume count=50) - * 30 null 10 blocks 30 to 39 - * - * null null 50 last 50 blocks? so if max(blocks.height) is 200, then blocks 151 to 200 - * null 200 null blocks 150 to 199 (excludes end block, assume count=50) - * null 200 10 blocks 190 to 199 (excludes end block) - */ - - if (startHeight != null && endHeight != null) { - sql.append("FROM Blocks "); - sql.append("WHERE height BETWEEN ? AND ?"); - bindParams.add(startHeight); - bindParams.add(Integer.valueOf(endHeight - 1)); - } else if (endHeight != null || (startHeight == null && count != null)) { - // we are going to return blocks from the end of the chain - if (count == null) - count = 50; - - if (endHeight == null) { - sql.append("FROM (SELECT height FROM Blocks ORDER BY height DESC LIMIT 1) AS MaxHeights (max_height) "); - sql.append("JOIN Blocks ON height BETWEEN (max_height - ? + 1) AND max_height "); - bindParams.add(count); - } else { - sql.append("FROM Blocks "); - sql.append("WHERE height BETWEEN ? AND ?"); - bindParams.add(Integer.valueOf(endHeight - count)); - bindParams.add(Integer.valueOf(endHeight - 1)); - } - - } else { - // we are going to return blocks from the start of the chain - if (startHeight == null) - startHeight = 1; - - if (count == null) - count = 50; - - sql.append("FROM Blocks "); - sql.append("WHERE height BETWEEN ? AND ?"); - bindParams.add(startHeight); - bindParams.add(Integer.valueOf(startHeight + count - 1)); - } - - List blockSummaries = new ArrayList<>(); - - try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { - if (resultSet == null) - return blockSummaries; - - do { - byte[] signature = resultSet.getBytes(1); - int height = resultSet.getInt(2); - byte[] minterPublicKey = resultSet.getBytes(3); - int onlineAccountsCount = resultSet.getInt(4); - long timestamp = resultSet.getLong(5); - int transactionCount = resultSet.getInt(6); - - BlockSummaryData blockSummary = new BlockSummaryData(height, signature, minterPublicKey, onlineAccountsCount, - timestamp, transactionCount); - blockSummaries.add(blockSummary); - } while (resultSet.next()); - - return blockSummaries; - } catch (SQLException e) { - throw new DataException("Unable to fetch height-ranged block summaries from repository", e); - } - } - @Override public int getOnlineAccountsSignaturesTrimHeight() throws DataException { String sql = "SELECT online_signatures_trim_height FROM DatabaseInfo"; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java new file mode 100644 index 00000000..930da828 --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java @@ -0,0 +1,87 @@ +package org.qortal.repository.hsqldb; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.repository.BlockArchiveWriter; +import org.qortal.repository.DataException; +import org.qortal.repository.RepositoryManager; +import org.qortal.transform.TransformationException; + +import java.io.IOException; + +/** + * + * When switching to an archiving node, we need to archive most of the database contents. + * This involves copying its data into flat files. + * If we do this entirely as a background process, it is very slow and can interfere with syncing. + * However, if we take the approach of doing this in bulk, before starting up the rest of the + * processes, this makes it much faster and less invasive. + * + * From that point, the original background archiving process will run, but can be dialled right down + * so not to interfere with syncing. + * + */ + + +public class HSQLDBDatabaseArchiving { + + private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabaseArchiving.class); + + + public static boolean buildBlockArchive() throws DataException { + try (final HSQLDBRepository repository = (HSQLDBRepository)RepositoryManager.getRepository()) { + + // Only build the archive if we have never done so before + int archiveHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); + if (archiveHeight > 0) { + // Already archived + return false; + } + + LOGGER.info("Building block archive - this process could take a while... (approx. 15 mins on high spec)"); + + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository, false); + int startHeight = 0; + + while (!Controller.isStopping()) { + try { + BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository); + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + switch (result) { + case OK: + // Increment block archive height + startHeight += writer.getWrittenCount(); + repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight); + repository.saveChanges(); + break; + + case STOPPING: + return false; + + case BLOCK_LIMIT_REACHED: + case NOT_ENOUGH_BLOCKS: + // We've reached the limit of the blocks we can archive + // Return from the whole method + return true; + + case BLOCK_NOT_FOUND: + // We tried to archive a block that didn't exist. This is a major failure and likely means + // that a bootstrap or re-sync is needed. Return rom the method + LOGGER.info("Error: block not found when building archive. If this error persists, " + + "a bootstrap or re-sync may be needed."); + return false; + } + + } catch (IOException | TransformationException | InterruptedException e) { + LOGGER.info("Caught exception when creating block cache", e); + return false; + } + } + } + + // If we got this far then something went wrong (most likely the app is stopping) + return false; + } + +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java index ba170bf6..969c954c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; import org.qortal.data.block.BlockData; +import org.qortal.repository.BlockArchiveWriter; import org.qortal.repository.DataException; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; @@ -36,6 +37,7 @@ public class HSQLDBDatabasePruning { private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabasePruning.class); + public static boolean pruneATStates() throws SQLException, DataException { try (final HSQLDBRepository repository = (HSQLDBRepository)RepositoryManager.getRepository()) { @@ -46,7 +48,18 @@ public class HSQLDBDatabasePruning { return false; } - LOGGER.info("Starting bulk prune of AT states - this process could take a while... (approx. 2 mins on high spec)"); + if (Settings.getInstance().isArchiveEnabled()) { + // Only proceed if we can see that the archiver has already finished + // This way, if the archiver failed for any reason, we can prune once it has had + // some opportunities to try again + boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository, false); + if (!upToDate) { + return false; + } + } + + LOGGER.info("Starting bulk prune of AT states - this process could take a while... " + + "(approx. 2 mins on high spec, or upwards of 30 mins in some cases)"); // Create new AT-states table to hold smaller dataset repository.executeCheckedUpdate("DROP TABLE IF EXISTS ATStatesNew"); @@ -68,11 +81,17 @@ public class HSQLDBDatabasePruning { // Calculate some constants for later use final int blockchainHeight = latestBlock.getHeight(); - final int maximumBlockToTrim = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + int maximumBlockToTrim = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + if (Settings.getInstance().isArchiveEnabled()) { + // Archive mode - don't prune anything that hasn't been archived yet + maximumBlockToTrim = Math.min(maximumBlockToTrim, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1); + } final int startHeight = maximumBlockToTrim; final int endHeight = blockchainHeight; final int blockStep = 10000; + + // Loop through all the LatestATStates and copy them to the new table LOGGER.info("Copying AT states..."); for (int height = 0; height < endHeight; height += blockStep) { @@ -99,7 +118,7 @@ public class HSQLDBDatabasePruning { } if (height >= startHeight) { - // Now copy this AT states for each recent block it is present in + // Now copy this AT's states for each recent block they is present in for (int i = startHeight; i < endHeight; i++) { if (latestAtHeight < i) { // This AT finished before this block so there is nothing to copy @@ -159,20 +178,25 @@ public class HSQLDBDatabasePruning { private static boolean pruneATStateData() throws SQLException, DataException { try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) { + if (Settings.getInstance().isArchiveEnabled()) { + // Don't prune ATStatesData in archive mode + return true; + } + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); if (latestBlock == null) { LOGGER.info("Unable to determine blockchain height, necessary for bulk ATStatesData pruning"); return false; } final int blockchainHeight = latestBlock.getHeight(); - final int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); // ATStateData is already trimmed - so carry on from where we left off in the past int pruneStartHeight = repository.getATRepository().getAtTrimHeight(); LOGGER.info("Starting bulk prune of AT states data - this process could take a while... (approx. 3 mins on high spec)"); while (pruneStartHeight < upperPrunableHeight) { - // Prune all AT state data up until our latest minus pruneBlockLimit + // Prune all AT state data up until our latest minus pruneBlockLimit (or our archive height) if (Controller.isStopping()) { return false; @@ -225,15 +249,30 @@ public class HSQLDBDatabasePruning { return false; } + if (Settings.getInstance().isArchiveEnabled()) { + // Only proceed if we can see that the archiver has already finished + // This way, if the archiver failed for any reason, we can prune once it has had + // some opportunities to try again + boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository, false); + if (!upToDate) { + return false; + } + } + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); if (latestBlock == null) { LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning"); return false; } final int blockchainHeight = latestBlock.getHeight(); - final int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); int pruneStartHeight = 0; + if (Settings.getInstance().isArchiveEnabled()) { + // Archive mode - don't prune anything that hasn't been archived yet + upperPrunableHeight = Math.min(upperPrunableHeight, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1); + } + LOGGER.info("Starting bulk prune of blocks - this process could take a while... (approx. 5 mins on high spec)"); while (pruneStartHeight < upperPrunableHeight) { diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index d696351f..66fe9029 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -873,6 +873,25 @@ public class HSQLDBDatabaseUpdates { stmt.execute("ALTER TABLE DatabaseInfo ADD block_prune_height INT NOT NULL DEFAULT 0"); break; + case 36: + // Block archive support + stmt.execute("ALTER TABLE DatabaseInfo ADD block_archive_height INT NOT NULL DEFAULT 0"); + + // Block archive (lookup table to map signature to height) + // Actual data is stored in archive files outside of the database + stmt.execute("CREATE TABLE BlockArchive (signature BlockSignature, height INTEGER NOT NULL, " + + "minted_when EpochMillis NOT NULL, minter QortalPublicKey NOT NULL, " + + "PRIMARY KEY (signature))"); + // For finding blocks by height. + stmt.execute("CREATE INDEX BlockArchiveHeightIndex ON BlockArchive (height)"); + // For finding blocks by the account that minted them. + stmt.execute("CREATE INDEX BlockArchiveMinterIndex ON BlockArchive (minter)"); + // For finding blocks by timestamp or finding height of latest block immediately before timestamp, etc. + stmt.execute("CREATE INDEX BlockArchiveTimestampHeightIndex ON BlockArchive (minted_when, height)"); + // Use a separate table space as this table will be very large. + stmt.execute("SET TABLE BlockArchive NEW SPACE"); + break; + default: // nothing to do return false; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 3a947cd6..6807c100 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -31,22 +31,7 @@ import org.qortal.crypto.Crypto; import org.qortal.data.crosschain.TradeBotData; import org.qortal.globalization.Translator; import org.qortal.gui.SysTray; -import org.qortal.repository.ATRepository; -import org.qortal.repository.AccountRepository; -import org.qortal.repository.ArbitraryRepository; -import org.qortal.repository.AssetRepository; -import org.qortal.repository.BlockRepository; -import org.qortal.repository.ChatRepository; -import org.qortal.repository.CrossChainRepository; -import org.qortal.repository.DataException; -import org.qortal.repository.GroupRepository; -import org.qortal.repository.MessageRepository; -import org.qortal.repository.NameRepository; -import org.qortal.repository.NetworkRepository; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.repository.TransactionRepository; -import org.qortal.repository.VotingRepository; +import org.qortal.repository.*; import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository; import org.qortal.settings.Settings; import org.qortal.utils.Base58; @@ -76,6 +61,7 @@ public class HSQLDBRepository implements Repository { private final ArbitraryRepository arbitraryRepository = new HSQLDBArbitraryRepository(this); private final AssetRepository assetRepository = new HSQLDBAssetRepository(this); private final BlockRepository blockRepository = new HSQLDBBlockRepository(this); + private final BlockArchiveRepository blockArchiveRepository = new HSQLDBBlockArchiveRepository(this); private final ChatRepository chatRepository = new HSQLDBChatRepository(this); private final CrossChainRepository crossChainRepository = new HSQLDBCrossChainRepository(this); private final GroupRepository groupRepository = new HSQLDBGroupRepository(this); @@ -143,6 +129,11 @@ public class HSQLDBRepository implements Repository { return this.blockRepository; } + @Override + public BlockArchiveRepository getBlockArchiveRepository() { + return this.blockArchiveRepository; + } + @Override public ChatRepository getChatRepository() { return this.chatRepository; diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 6ac7342c..6527d7e0 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -129,6 +129,15 @@ public class Settings { private int blockPruneBatchSize = 10000; // blocks + /** Whether we should archive old data to reduce the database size */ + private boolean archiveEnabled = true; + /** How often to attempt archiving (ms). */ + private long archiveInterval = 7171L; // milliseconds + /** The maximum number of blocks that can exist in both the + * database and the archive at the same time */ + private int maxDuplicatedBlocksWhenArchiving = 100000; + + // Peer-to-peer related private boolean isTestNet = false; /** Port number for inbound peer-to-peer connections. */ @@ -574,4 +583,17 @@ public class Settings { return this.blockPruneBatchSize; } + + public boolean isArchiveEnabled() { + return this.archiveEnabled; + } + + public long getArchiveInterval() { + return this.archiveInterval; + } + + public int getMaxDuplicatedBlocksWhenArchiving() { + return this.maxDuplicatedBlocksWhenArchiving; + } + } From 0657ca2969f4157718c2c8940795a2dec14486fe Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 9 Sep 2021 17:46:19 +0100 Subject: [PATCH 17/22] atStatesMaxLifetime increased to 5 days For now, we need some headroom to allow for orphaning in the event of a problem. Orphaning currently fails if there is no ATStatesData available (which is the case for trimmed blocks). This could ultimately be solved by retaining older unique states. --- src/main/java/org/qortal/settings/Settings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 6527d7e0..98521646 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -94,7 +94,7 @@ public class Settings { private int blockCacheSize = 10; /** How long to keep old, full, AT state data (ms). */ - private long atStatesMaxLifetime = 24 * 60 * 60 * 1000L; // milliseconds + private long atStatesMaxLifetime = 5 * 24 * 60 * 60 * 1000L; // milliseconds /** How often to attempt AT state trimming (ms). */ private long atStatesTrimInterval = 5678L; // milliseconds /** Block height range to scan for trimmable AT states.
From 14acc4feb933e13dd6a341223cc63b4b0d9cecec Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 12 Sep 2021 09:52:28 +0100 Subject: [PATCH 18/22] Removed maxDuplicatedBlocksWhenArchiving setting as it's no longer needed. --- .../controller/repository/BlockArchiver.java | 2 +- .../qortal/repository/BlockArchiveWriter.java | 26 +++---------------- .../hsqldb/HSQLDBDatabaseArchiving.java | 2 +- .../hsqldb/HSQLDBDatabasePruning.java | 4 +-- .../java/org/qortal/settings/Settings.java | 7 ----- 5 files changed, 8 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/qortal/controller/repository/BlockArchiver.java b/src/main/java/org/qortal/controller/repository/BlockArchiver.java index f7bafe7d..aab4b4fa 100644 --- a/src/main/java/org/qortal/controller/repository/BlockArchiver.java +++ b/src/main/java/org/qortal/controller/repository/BlockArchiver.java @@ -35,7 +35,7 @@ public class BlockArchiver implements Runnable { while (!Controller.isStopping()) { repository.discardChanges(); - final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository, true); + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); Thread.sleep(Settings.getInstance().getArchiveInterval()); diff --git a/src/main/java/org/qortal/repository/BlockArchiveWriter.java b/src/main/java/org/qortal/repository/BlockArchiveWriter.java index 4aeb1a32..59d07072 100644 --- a/src/main/java/org/qortal/repository/BlockArchiveWriter.java +++ b/src/main/java/org/qortal/repository/BlockArchiveWriter.java @@ -42,34 +42,16 @@ public class BlockArchiveWriter { this.repository = repository; } - public static int getMaxArchiveHeight(Repository repository, boolean useMaximumDuplicatedLimit) throws DataException { + public static int getMaxArchiveHeight(Repository repository) throws DataException { // We must only archive trimmed blocks, or the archive will grow far too large final int accountSignaturesTrimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight(); final int atTrimStartHeight = repository.getATRepository().getAtTrimHeight(); final int trimStartHeight = Math.min(accountSignaturesTrimStartHeight, atTrimStartHeight); - - // In some cases we want to restrict the upper height of the archiver to save space - if (useMaximumDuplicatedLimit) { - // To save on disk space, it's best to not allow the archiver to get too far ahead of the pruner - // This reduces the amount of data that is held twice during the transition - final int blockPruneStartHeight = repository.getBlockRepository().getBlockPruneHeight(); - final int atPruneStartHeight = repository.getATRepository().getAtPruneHeight(); - final int pruneStartHeight = Math.min(blockPruneStartHeight, atPruneStartHeight); - final int maximumDuplicatedBlocks = Settings.getInstance().getMaxDuplicatedBlocksWhenArchiving(); - - // To summarize the above: - // - We must never archive anything greater than or equal to trimStartHeight - // - We should avoid archiving anything maximumDuplicatedBlocks higher than pruneStartHeight - return Math.min(trimStartHeight, pruneStartHeight + maximumDuplicatedBlocks); - } - else { - // We don't want to apply the maximum duplicated limit - return trimStartHeight; - } + return trimStartHeight - 1; // subtract 1 because these values represent the first _untrimmed_ block } - public static boolean isArchiverUpToDate(Repository repository, boolean useMaximumDuplicatedLimit) throws DataException { - final int maxArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository, useMaximumDuplicatedLimit); + public static boolean isArchiverUpToDate(Repository repository) throws DataException { + final int maxArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); final int actualArchiveHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); final float progress = (float)actualArchiveHeight / (float) maxArchiveHeight; LOGGER.info(String.format("maxArchiveHeight: %d, actualArchiveHeight: %d, progress: %f", diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java index 930da828..7a7b66f3 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java @@ -41,7 +41,7 @@ public class HSQLDBDatabaseArchiving { LOGGER.info("Building block archive - this process could take a while... (approx. 15 mins on high spec)"); - final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository, false); + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); int startHeight = 0; while (!Controller.isStopping()) { diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java index 969c954c..65139743 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java @@ -52,7 +52,7 @@ public class HSQLDBDatabasePruning { // Only proceed if we can see that the archiver has already finished // This way, if the archiver failed for any reason, we can prune once it has had // some opportunities to try again - boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository, false); + boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository); if (!upToDate) { return false; } @@ -253,7 +253,7 @@ public class HSQLDBDatabasePruning { // Only proceed if we can see that the archiver has already finished // This way, if the archiver failed for any reason, we can prune once it has had // some opportunities to try again - boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository, false); + boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository); if (!upToDate) { return false; } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 98521646..f6c89e61 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -133,9 +133,6 @@ public class Settings { private boolean archiveEnabled = true; /** How often to attempt archiving (ms). */ private long archiveInterval = 7171L; // milliseconds - /** The maximum number of blocks that can exist in both the - * database and the archive at the same time */ - private int maxDuplicatedBlocksWhenArchiving = 100000; // Peer-to-peer related @@ -592,8 +589,4 @@ public class Settings { return this.archiveInterval; } - public int getMaxDuplicatedBlocksWhenArchiving() { - return this.maxDuplicatedBlocksWhenArchiving; - } - } From 2a36b83dea882da414313275544aebc17db546e0 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 12 Sep 2021 09:55:49 +0100 Subject: [PATCH 19/22] Removed BLOCK_LIMIT_REACHED result from the block archive writer. This wasn't needed, and is now instead caught by the NOT_ENOUGH_BLOCKS result. --- .../java/org/qortal/controller/repository/BlockArchiver.java | 1 - src/main/java/org/qortal/repository/BlockArchiveWriter.java | 3 +-- .../org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/repository/BlockArchiver.java b/src/main/java/org/qortal/controller/repository/BlockArchiver.java index aab4b4fa..d6860347 100644 --- a/src/main/java/org/qortal/controller/repository/BlockArchiver.java +++ b/src/main/java/org/qortal/controller/repository/BlockArchiver.java @@ -71,7 +71,6 @@ public class BlockArchiver implements Runnable { case STOPPING: return; - case BLOCK_LIMIT_REACHED: // We've reached the limit of the blocks we can archive // Sleep for a while to allow more to become available case NOT_ENOUGH_BLOCKS: diff --git a/src/main/java/org/qortal/repository/BlockArchiveWriter.java b/src/main/java/org/qortal/repository/BlockArchiveWriter.java index 59d07072..efef689e 100644 --- a/src/main/java/org/qortal/repository/BlockArchiveWriter.java +++ b/src/main/java/org/qortal/repository/BlockArchiveWriter.java @@ -24,7 +24,6 @@ public class BlockArchiveWriter { OK, STOPPING, NOT_ENOUGH_BLOCKS, - BLOCK_LIMIT_REACHED, BLOCK_NOT_FOUND } @@ -99,7 +98,7 @@ public class BlockArchiveWriter { int currentHeight = startHeight + i; if (currentHeight >= endHeight) { - return BlockArchiveWriteResult.BLOCK_LIMIT_REACHED; + break; } //LOGGER.info("Fetching block {}...", currentHeight); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java index 7a7b66f3..618d5115 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java @@ -59,7 +59,6 @@ public class HSQLDBDatabaseArchiving { case STOPPING: return false; - case BLOCK_LIMIT_REACHED: case NOT_ENOUGH_BLOCKS: // We've reached the limit of the blocks we can archive // Return from the whole method From 6a55b052f5aaf2173a23edf52322572bac910c42 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 12 Sep 2021 09:57:12 +0100 Subject: [PATCH 20/22] Fixed some bugs found in unit testing. --- src/main/java/org/qortal/repository/BlockArchiveWriter.java | 2 +- .../java/org/qortal/repository/hsqldb/HSQLDBATRepository.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/repository/BlockArchiveWriter.java b/src/main/java/org/qortal/repository/BlockArchiveWriter.java index efef689e..11151e17 100644 --- a/src/main/java/org/qortal/repository/BlockArchiveWriter.java +++ b/src/main/java/org/qortal/repository/BlockArchiveWriter.java @@ -97,7 +97,7 @@ public class BlockArchiveWriter { } int currentHeight = startHeight + i; - if (currentHeight >= endHeight) { + if (currentHeight > endHeight) { break; } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index e0baa136..56658ec7 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -744,7 +744,7 @@ public class HSQLDBATRepository implements ATRepository { int deletedCount = 0; - for (int height = minHeight; height < maxHeight; height++) { + for (int height = minHeight; height <= maxHeight; height++) { // Give up if we're stopping if (Controller.isStopping()) { From 1d8351f921d9200f6cedebeee7161187b4f03bb8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 12 Sep 2021 10:10:25 +0100 Subject: [PATCH 21/22] Added importFromArchive() feature This allows archived blocks to be imported back into HSQLDB in order to make them SQL-compatible again. --- .../qortal/repository/BlockArchiveReader.java | 15 ++++ .../repository/BlockArchiveRepository.java | 12 +++ .../hsqldb/HSQLDBBlockArchiveRepository.java | 15 ++++ .../org/qortal/utils/BlockArchiveUtils.java | 78 +++++++++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 src/main/java/org/qortal/utils/BlockArchiveUtils.java diff --git a/src/main/java/org/qortal/repository/BlockArchiveReader.java b/src/main/java/org/qortal/repository/BlockArchiveReader.java index 1b68a7c5..081917b2 100644 --- a/src/main/java/org/qortal/repository/BlockArchiveReader.java +++ b/src/main/java/org/qortal/repository/BlockArchiveReader.java @@ -105,6 +105,21 @@ public class BlockArchiveReader { return null; } + public List, List>> fetchBlocksFromRange( + int startHeight, int endHeight) { + + List, List>> blockInfoList = new ArrayList<>(); + + for (int height = startHeight; height <= endHeight; height++) { + Triple, List> blockInfo = this.fetchBlockAtHeight(height); + if (blockInfo == null) { + return blockInfoList; + } + blockInfoList.add(blockInfo); + } + return blockInfoList; + } + public Integer fetchHeightForSignature(byte[] signature, Repository repository) { // Lookup the height for the requested signature try { diff --git a/src/main/java/org/qortal/repository/BlockArchiveRepository.java b/src/main/java/org/qortal/repository/BlockArchiveRepository.java index c702a7ef..45465e93 100644 --- a/src/main/java/org/qortal/repository/BlockArchiveRepository.java +++ b/src/main/java/org/qortal/repository/BlockArchiveRepository.java @@ -36,6 +36,18 @@ public interface BlockArchiveRepository { */ public BlockData fromHeight(int height) throws DataException; + /** + * Returns a list of BlockData objects from archive using + * block height range. + * + * @param startHeight + * @return a list of BlockData objects, or an empty list if + * not found in blockchain. It is not guaranteed that all + * requested blocks will be returned. + * @throws DataException + */ + public List fromRange(int startHeight, int endHeight) throws DataException; + /** * Returns BlockData from archive using block reference. * Currently relies on a child block being the one block diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java index c491f862..32270213 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java @@ -3,6 +3,7 @@ package org.qortal.repository.hsqldb; import org.qortal.api.ApiError; import org.qortal.api.ApiExceptionFactory; import org.qortal.api.model.BlockSignerSummary; +import org.qortal.block.Block; import org.qortal.data.block.BlockArchiveData; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; @@ -53,6 +54,20 @@ public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository { return null; } + @Override + public List fromRange(int startHeight, int endHeight) throws DataException { + List blocks = new ArrayList<>(); + + for (int height = startHeight; height < endHeight; height++) { + BlockData blockData = this.fromHeight(height); + if (blockData == null) { + return blocks; + } + blocks.add(blockData); + } + return blocks; + } + @Override public BlockData fromReference(byte[] reference) throws DataException { BlockData referenceBlock = this.repository.getBlockArchiveRepository().fromSignature(reference); diff --git a/src/main/java/org/qortal/utils/BlockArchiveUtils.java b/src/main/java/org/qortal/utils/BlockArchiveUtils.java new file mode 100644 index 00000000..0beff026 --- /dev/null +++ b/src/main/java/org/qortal/utils/BlockArchiveUtils.java @@ -0,0 +1,78 @@ +package org.qortal.utils; + +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.BlockArchiveReader; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; + +import java.util.List; + +public class BlockArchiveUtils { + + /** + * importFromArchive + *

+ * Reads the requested block range from the archive + * and imports the BlockData and AT state data hashes + * This can be used to convert a block archive back + * into the HSQLDB, in order to make it SQL-compatible + * again. + *

+ * Note: calls discardChanges() and saveChanges(), so + * make sure that you commit any existing repository + * changes before calling this method. + * + * @param startHeight The earliest block to import + * @param endHeight The latest block to import + * @param repository A clean repository session + * @throws DataException + */ + public static void importFromArchive(int startHeight, int endHeight, Repository repository) throws DataException { + repository.discardChanges(); + final int requestedRange = endHeight+1-startHeight; + + List, List>> blockInfoList = + BlockArchiveReader.getInstance().fetchBlocksFromRange(startHeight, endHeight); + + // Ensure that we have received all of the requested blocks + if (blockInfoList == null || blockInfoList.isEmpty()) { + throw new IllegalStateException("No blocks found when importing from archive"); + } + if (blockInfoList.size() != requestedRange) { + throw new IllegalStateException("Non matching block count when importing from archive"); + } + Triple, List> firstBlock = blockInfoList.get(0); + if (firstBlock == null || firstBlock.getA().getHeight() != startHeight) { + throw new IllegalStateException("Non matching first block when importing from archive"); + } + if (blockInfoList.size() > 0) { + Triple, List> lastBlock = + blockInfoList.get(blockInfoList.size() - 1); + if (lastBlock == null || lastBlock.getA().getHeight() != endHeight) { + throw new IllegalStateException("Non matching last block when importing from archive"); + } + } + + // Everything seems okay, so go ahead with the import + for (Triple, List> blockInfo : blockInfoList) { + try { + // Save block + repository.getBlockRepository().save(blockInfo.getA()); + + // Save AT state data hashes + for (ATStateData atStateData : blockInfo.getC()) { + atStateData.setHeight(blockInfo.getA().getHeight()); + repository.getATRepository().save(atStateData); + } + + } catch (DataException e) { + repository.discardChanges(); + throw new IllegalStateException("Unable to import blocks from archive"); + } + } + repository.saveChanges(); + } + +} From 14f6fd19ef825c92271183b4db5b8096ad71b411 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 12 Sep 2021 10:13:52 +0100 Subject: [PATCH 22/22] Added unit tests for trimming, pruning, and archiving. --- .../org/qortal/controller/BlockMinter.java | 3 +- .../qortal/repository/BlockArchiveWriter.java | 25 +- .../org/qortal/test/BlockArchiveTests.java | 500 ++++++++++++++++++ src/test/java/org/qortal/test/PruneTests.java | 91 ++++ .../org/qortal/test/at/AtRepositoryTests.java | 156 +++--- .../java/org/qortal/test/common/AtUtils.java | 81 +++ .../test-settings-v2-block-archive.json | 11 + 7 files changed, 780 insertions(+), 87 deletions(-) create mode 100644 src/test/java/org/qortal/test/BlockArchiveTests.java create mode 100644 src/test/java/org/qortal/test/PruneTests.java create mode 100644 src/test/java/org/qortal/test/common/AtUtils.java create mode 100644 src/test/resources/test-settings-v2-block-archive.json diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 8b6563f2..318b1ac2 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -421,7 +421,8 @@ public class BlockMinter extends Thread { // Add to blockchain newBlock.process(); - LOGGER.info(String.format("Minted new test block: %d", newBlock.getBlockData().getHeight())); + LOGGER.info(String.format("Minted new test block: %d sig: %.8s", + newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature()))); repository.saveChanges(); diff --git a/src/main/java/org/qortal/repository/BlockArchiveWriter.java b/src/main/java/org/qortal/repository/BlockArchiveWriter.java index 11151e17..77c98d96 100644 --- a/src/main/java/org/qortal/repository/BlockArchiveWriter.java +++ b/src/main/java/org/qortal/repository/BlockArchiveWriter.java @@ -33,7 +33,11 @@ public class BlockArchiveWriter { private final int endHeight; private final Repository repository; + private long fileSizeTarget = 100 * 1024 * 1024; // 100MiB + private boolean shouldEnforceFileSizeTarget = true; + private int writtenCount; + private Path outputPath; public BlockArchiveWriter(int startHeight, int endHeight, Repository repository) { this.startHeight = startHeight; @@ -87,8 +91,9 @@ public class BlockArchiveWriter { LOGGER.info(String.format("Fetching blocks from height %d...", startHeight)); int i = 0; - long fileSizeTarget = 100 * 1024 * 1024; // 100MiB - while (headerBytes.size() + bytes.size() < fileSizeTarget) { + while (headerBytes.size() + bytes.size() < this.fileSizeTarget + || this.shouldEnforceFileSizeTarget == false) { + if (Controller.isStopping()) { return BlockArchiveWriteResult.STOPPING; } @@ -132,7 +137,7 @@ public class BlockArchiveWriter { LOGGER.info(String.format("Total length of %d blocks is %d bytes", i, totalLength)); // Validate file size, in case something went wrong - if (totalLength < fileSizeTarget) { + if (totalLength < fileSizeTarget && this.shouldEnforceFileSizeTarget) { return BlockArchiveWriteResult.NOT_ENOUGH_BLOCKS; } @@ -164,6 +169,7 @@ public class BlockArchiveWriter { BlockArchiveReader.getInstance().invalidateFileListCache(); this.writtenCount = i; + this.outputPath = Paths.get(filePath); return BlockArchiveWriteResult.OK; } @@ -171,4 +177,17 @@ public class BlockArchiveWriter { return this.writtenCount; } + public Path getOutputPath() { + return this.outputPath; + } + + public void setFileSizeTarget(long fileSizeTarget) { + this.fileSizeTarget = fileSizeTarget; + } + + // For testing, to avoid having to pre-calculate file sizes + public void setShouldEnforceFileSizeTarget(boolean shouldEnforceFileSizeTarget) { + this.shouldEnforceFileSizeTarget = shouldEnforceFileSizeTarget; + } + } diff --git a/src/test/java/org/qortal/test/BlockArchiveTests.java b/src/test/java/org/qortal/test/BlockArchiveTests.java new file mode 100644 index 00000000..c05915cd --- /dev/null +++ b/src/test/java/org/qortal/test/BlockArchiveTests.java @@ -0,0 +1,500 @@ +package org.qortal.test; + +import org.apache.commons.io.FileUtils; +import org.ciyam.at.CompilationException; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.controller.BlockMinter; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.*; +import org.qortal.settings.Settings; +import org.qortal.test.common.AtUtils; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.transform.TransformationException; +import org.qortal.utils.BlockArchiveUtils; +import org.qortal.utils.Triple; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +public class BlockArchiveTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); // Necessary to set NTP offset + Common.useSettings("test-settings-v2-block-archive.json"); + this.deleteArchiveDirectory(); + } + + @After + public void afterTest() throws DataException { + this.deleteArchiveDirectory(); + } + + + @Test + public void testWriter() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(900, maximumArchiveHeight); + + // Write blocks 2-900 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(900 - 1, writer.getWrittenCount()); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + } + } + + @Test + public void testWriterAndReader() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(900, maximumArchiveHeight); + + // Write blocks 2-900 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(900 - 1, writer.getWrittenCount()); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + + // Read block 2 from the archive + BlockArchiveReader reader = BlockArchiveReader.getInstance(); + Triple, List> block2Info = reader.fetchBlockAtHeight(2); + BlockData block2ArchiveData = block2Info.getA(); + + // Read block 2 from the repository + BlockData block2RepositoryData = repository.getBlockRepository().fromHeight(2); + + // Ensure the values match + assertEquals(block2ArchiveData.getHeight(), block2RepositoryData.getHeight()); + assertArrayEquals(block2ArchiveData.getSignature(), block2RepositoryData.getSignature()); + + // Test some values in the archive + assertEquals(1, block2ArchiveData.getOnlineAccountsCount()); + + // Read block 900 from the archive + Triple, List> block900Info = reader.fetchBlockAtHeight(900); + BlockData block900ArchiveData = block900Info.getA(); + + // Read block 900 from the repository + BlockData block900RepositoryData = repository.getBlockRepository().fromHeight(900); + + // Ensure the values match + assertEquals(block900ArchiveData.getHeight(), block900RepositoryData.getHeight()); + assertArrayEquals(block900ArchiveData.getSignature(), block900RepositoryData.getSignature()); + + // Test some values in the archive + assertEquals(1, block900ArchiveData.getOnlineAccountsCount()); + + } + } + + @Test + public void testArchivedAtStates() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 10; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // 9 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(10); + repository.getATRepository().setAtTrimHeight(10); + + // Check the max archive height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(9, maximumArchiveHeight); + + // Write blocks 2-9 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(9 - 1, writer.getWrittenCount()); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(9 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + + // Check blocks 3-9 + for (Integer testHeight = 2; testHeight <= 9; testHeight++) { + + // Read a block from the archive + BlockArchiveReader reader = BlockArchiveReader.getInstance(); + Triple, List> blockInfo = reader.fetchBlockAtHeight(testHeight); + BlockData archivedBlockData = blockInfo.getA(); + ATStateData archivedAtStateData = blockInfo.getC().isEmpty() ? null : blockInfo.getC().get(0); + List archivedTransactions = blockInfo.getB(); + + // Read the same block from the repository + BlockData repositoryBlockData = repository.getBlockRepository().fromHeight(testHeight); + ATStateData repositoryAtStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight); + + // Ensure the repository has full AT state data + assertNotNull(repositoryAtStateData.getStateHash()); + assertNotNull(repositoryAtStateData.getStateData()); + + // Check the archived AT state + if (testHeight == 2) { + // Block 2 won't have an AT state hash because it's initial (and has the DEPLOY_AT in the same block) + assertNull(archivedAtStateData); + + assertEquals(1, archivedTransactions.size()); + assertEquals(Transaction.TransactionType.DEPLOY_AT, archivedTransactions.get(0).getType()); + } + else { + // For blocks 3+, ensure the archive has the AT state data, but not the hashes + assertNotNull(archivedAtStateData.getStateHash()); + assertNull(archivedAtStateData.getStateData()); + + // They also shouldn't have any transactions + assertTrue(archivedTransactions.isEmpty()); + } + + // Also check the online accounts count and height + assertEquals(1, archivedBlockData.getOnlineAccountsCount()); + assertEquals(testHeight, archivedBlockData.getHeight()); + + // Ensure the values match + assertEquals(archivedBlockData.getHeight(), repositoryBlockData.getHeight()); + assertArrayEquals(archivedBlockData.getSignature(), repositoryBlockData.getSignature()); + assertEquals(archivedBlockData.getOnlineAccountsCount(), repositoryBlockData.getOnlineAccountsCount()); + assertArrayEquals(archivedBlockData.getMinterSignature(), repositoryBlockData.getMinterSignature()); + assertEquals(archivedBlockData.getATCount(), repositoryBlockData.getATCount()); + assertEquals(archivedBlockData.getOnlineAccountsCount(), repositoryBlockData.getOnlineAccountsCount()); + assertArrayEquals(archivedBlockData.getReference(), repositoryBlockData.getReference()); + assertEquals(archivedBlockData.getTimestamp(), repositoryBlockData.getTimestamp()); + assertEquals(archivedBlockData.getATFees(), repositoryBlockData.getATFees()); + assertEquals(archivedBlockData.getTotalFees(), repositoryBlockData.getTotalFees()); + assertEquals(archivedBlockData.getTransactionCount(), repositoryBlockData.getTransactionCount()); + assertArrayEquals(archivedBlockData.getTransactionsSignature(), repositoryBlockData.getTransactionsSignature()); + + if (testHeight != 2) { + assertArrayEquals(archivedAtStateData.getStateHash(), repositoryAtStateData.getStateHash()); + } + } + + // Check block 10 (unarchived) + BlockArchiveReader reader = BlockArchiveReader.getInstance(); + Triple, List> blockInfo = reader.fetchBlockAtHeight(10); + assertNull(blockInfo); + + } + + } + + @Test + public void testArchiveAndPrune() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Assume 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(900, maximumArchiveHeight); + + // Write blocks 2-900 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(900 - 1, writer.getWrittenCount()); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + + // Ensure the SQL repository contains blocks 2 and 900... + assertNotNull(repository.getBlockRepository().fromHeight(2)); + assertNotNull(repository.getBlockRepository().fromHeight(900)); + + // Prune all the archived blocks + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 900); + assertEquals(900-1, numBlocksPruned); + repository.getBlockRepository().setBlockPruneHeight(901); + + // Prune the AT states for the archived blocks + repository.getATRepository().rebuildLatestAtStates(); + int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 900); + assertEquals(900-1, numATStatesPruned); + repository.getATRepository().setAtPruneHeight(901); + + // Now ensure the SQL repository is missing blocks 2 and 900... + assertNull(repository.getBlockRepository().fromHeight(2)); + assertNull(repository.getBlockRepository().fromHeight(900)); + + // ... but it's not missing blocks 1 and 901 (we don't prune the genesis block) + assertNotNull(repository.getBlockRepository().fromHeight(1)); + assertNotNull(repository.getBlockRepository().fromHeight(901)); + + // Validate the latest block height in the repository + assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + } + } + + @Test + public void testTrimArchivePruneAndOrphan() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Make sure that block 500 has full AT state data and data hash + List block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500); + ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + + // Trim the first 500 blocks + repository.getBlockRepository().trimOldOnlineAccountsSignatures(0, 500); + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(501); + repository.getATRepository().trimAtStates(0, 500, 1000); + repository.getATRepository().setAtTrimHeight(501); + + // Now block 500 should only have the AT state data hash + block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500); + atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500); + assertNotNull(atStatesData.getStateHash()); + assertNull(atStatesData.getStateData()); + + // ... but block 501 should have the full data + List block501AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(501); + atStatesData = repository.getATRepository().getATStateAtHeight(block501AtStatesData.get(0).getATAddress(), 501); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(500, maximumArchiveHeight); + + BlockData block3DataPreArchive = repository.getBlockRepository().fromHeight(3); + + // Write blocks 2-500 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(500 - 1, writer.getWrittenCount()); // -1 for the genesis block + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(500 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + + // Ensure the SQL repository contains blocks 2 and 500... + assertNotNull(repository.getBlockRepository().fromHeight(2)); + assertNotNull(repository.getBlockRepository().fromHeight(500)); + + // Prune all the archived blocks + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 500); + assertEquals(500-1, numBlocksPruned); + repository.getBlockRepository().setBlockPruneHeight(501); + + // Prune the AT states for the archived blocks + repository.getATRepository().rebuildLatestAtStates(); + int numATStatesPruned = repository.getATRepository().pruneAtStates(2, 500); + assertEquals(499, numATStatesPruned); + repository.getATRepository().setAtPruneHeight(501); + + // Now ensure the SQL repository is missing blocks 2 and 500... + assertNull(repository.getBlockRepository().fromHeight(2)); + assertNull(repository.getBlockRepository().fromHeight(500)); + + // ... but it's not missing blocks 1 and 501 (we don't prune the genesis block) + assertNotNull(repository.getBlockRepository().fromHeight(1)); + assertNotNull(repository.getBlockRepository().fromHeight(501)); + + // Validate the latest block height in the repository + assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Now orphan some unarchived blocks. + BlockUtils.orphanBlocks(repository, 500); + assertEquals(502, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // We're close to the lower limit of the SQL database now, so + // we need to import some blocks from the archive + BlockArchiveUtils.importFromArchive(401, 500, repository); + + // Ensure the SQL repository now contains block 401 but not 400... + assertNotNull(repository.getBlockRepository().fromHeight(401)); + assertNull(repository.getBlockRepository().fromHeight(400)); + + // Import the remaining 399 blocks + BlockArchiveUtils.importFromArchive(2, 400, repository); + + // Verify that block 3 matches the original + BlockData block3DataPostArchive = repository.getBlockRepository().fromHeight(3); + assertArrayEquals(block3DataPreArchive.getSignature(), block3DataPostArchive.getSignature()); + assertEquals(block3DataPreArchive.getHeight(), block3DataPostArchive.getHeight()); + + // Orphan 1 more block, which should be the last one that is possible to be orphaned + BlockUtils.orphanBlocks(repository, 1); + + // Orphan another block, which should fail + Exception exception = null; + try { + BlockUtils.orphanBlocks(repository, 1); + } catch (DataException e) { + exception = e; + } + + // Ensure that a DataException is thrown because there is no more AT states data available + assertNotNull(exception); + assertEquals(DataException.class, exception.getClass()); + + // FUTURE: we may be able to retain unique AT states when trimming, to avoid this exception + // and allow orphaning back through blocks with trimmed AT states. + + } + } + + + private void deleteArchiveDirectory() { + // Delete archive directory if exists + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath(); + try { + FileUtils.deleteDirectory(archivePath.toFile()); + } catch (IOException e) { + + } + } + +} diff --git a/src/test/java/org/qortal/test/PruneTests.java b/src/test/java/org/qortal/test/PruneTests.java new file mode 100644 index 00000000..0914d794 --- /dev/null +++ b/src/test/java/org/qortal/test/PruneTests.java @@ -0,0 +1,91 @@ +package org.qortal.test; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.BlockMinter; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.AtUtils; +import org.qortal.test.common.Common; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +public class PruneTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testPruning() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks + for (int i = 2; i <= 10; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Make sure that all blocks have full AT state data and data hash + for (Integer i=2; i <= 10; i++) { + BlockData blockData = repository.getBlockRepository().fromHeight(i); + assertNotNull(blockData.getSignature()); + assertEquals(i, blockData.getHeight()); + List atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i); + assertNotNull(atStatesDataList); + assertFalse(atStatesDataList.isEmpty()); + ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(atStatesDataList.get(0).getATAddress(), i); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + } + + // Prune blocks 2-5 + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 5); + assertEquals(4, numBlocksPruned); + repository.getBlockRepository().setBlockPruneHeight(6); + + // Prune AT states for blocks 2-5 + int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 5); + assertEquals(4, numATStatesPruned); + repository.getATRepository().setAtPruneHeight(6); + + // Make sure that blocks 2-5 are now missing block data and AT states data + for (Integer i=2; i <= 5; i++) { + BlockData blockData = repository.getBlockRepository().fromHeight(i); + assertNull(blockData); + List atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i); + assertTrue(atStatesDataList.isEmpty()); + } + + // ... but blocks 6-10 have block data and full AT states data + for (Integer i=6; i <= 10; i++) { + BlockData blockData = repository.getBlockRepository().fromHeight(i); + assertNotNull(blockData.getSignature()); + List atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i); + assertNotNull(atStatesDataList); + assertFalse(atStatesDataList.isEmpty()); + ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(atStatesDataList.get(0).getATAddress(), i); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + } + } + } + +} diff --git a/src/test/java/org/qortal/test/at/AtRepositoryTests.java b/src/test/java/org/qortal/test/at/AtRepositoryTests.java index 0b302435..8ef4c774 100644 --- a/src/test/java/org/qortal/test/at/AtRepositoryTests.java +++ b/src/test/java/org/qortal/test/at/AtRepositoryTests.java @@ -21,6 +21,7 @@ import org.qortal.group.Group; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.AtUtils; import org.qortal.test.common.BlockUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; @@ -35,13 +36,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetATStateAtHeightWithData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -58,13 +59,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetATStateAtHeightWithoutData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -87,13 +88,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetLatestATStateWithData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -111,13 +112,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetLatestATStatePostTrimming() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -143,15 +144,67 @@ public class AtRepositoryTests extends Common { } } + @Test + public void testOrphanTrimmedATStates() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + + int blockchainHeight = repository.getBlockRepository().getBlockchainHeight(); + int maxTrimHeight = blockchainHeight - 4; + Integer testHeight = maxTrimHeight + 1; + + // Trim AT state data + repository.getATRepository().rebuildLatestAtStates(); + repository.saveChanges(); + repository.getATRepository().trimAtStates(2, maxTrimHeight, 1000); + + // Orphan 3 blocks + // This leaves one more untrimmed block, so the latest AT state should be available + BlockUtils.orphanBlocks(repository, 3); + + ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); + assertEquals(testHeight, atStateData.getHeight()); + + // We should always have the latest AT state data available + assertNotNull(atStateData.getStateData()); + + // Orphan 1 more block + Exception exception = null; + try { + BlockUtils.orphanBlocks(repository, 1); + } catch (DataException e) { + exception = e; + } + + // Ensure that a DataException is thrown because there is no more AT states data available + assertNotNull(exception); + assertEquals(DataException.class, exception.getClass()); + assertEquals(String.format("Can't find previous AT state data for %s", atAddress), exception.getMessage()); + + // FUTURE: we may be able to retain unique AT states when trimming, to avoid this exception + // and allow orphaning back through blocks with trimmed AT states. + } + } + @Test public void testGetMatchingFinalATStatesWithoutDataValue() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -191,13 +244,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetMatchingFinalATStatesWithDataValue() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -237,13 +290,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetBlockATStatesAtHeightWithData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - doDeploy(repository, deployer, creationBytes, fundingAmount); + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); // Mint a few blocks for (int i = 0; i < 10; ++i) @@ -264,13 +317,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetBlockATStatesAtHeightWithoutData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - doDeploy(repository, deployer, creationBytes, fundingAmount); + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); // Mint a few blocks for (int i = 0; i < 10; ++i) @@ -297,13 +350,13 @@ public class AtRepositoryTests extends Common { @Test public void testSaveATStateWithData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -328,13 +381,13 @@ public class AtRepositoryTests extends Common { @Test public void testSaveATStateWithoutData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -364,67 +417,4 @@ public class AtRepositoryTests extends Common { assertNull(atStateData.getStateData()); } } - - private byte[] buildSimpleAT() { - // Pretend we use 4 values in data segment - int addrCounter = 4; - - // Data segment - ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); - - ByteBuffer codeByteBuffer = ByteBuffer.allocate(512); - - // Two-pass version - for (int pass = 0; pass < 2; ++pass) { - codeByteBuffer.clear(); - - try { - // Stop and wait for next block - codeByteBuffer.put(OpCode.STP_IMD.compile()); - } catch (CompilationException e) { - throw new IllegalStateException("Unable to compile AT?", e); - } - } - - codeByteBuffer.flip(); - - byte[] codeBytes = new byte[codeByteBuffer.limit()]; - codeByteBuffer.get(codeBytes); - - final short ciyamAtVersion = 2; - final short numCallStackPages = 0; - final short numUserStackPages = 0; - final long minActivationAmount = 0L; - - return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); - } - - private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException { - long txTimestamp = System.currentTimeMillis(); - byte[] lastReference = deployer.getLastReference(); - - if (lastReference == null) { - System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); - System.exit(2); - } - - Long fee = null; - String name = "Test AT"; - String description = "Test AT"; - String atType = "Test"; - String tags = "TEST"; - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); - TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); - - DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); - - return deployAtTransaction; - } - } diff --git a/src/test/java/org/qortal/test/common/AtUtils.java b/src/test/java/org/qortal/test/common/AtUtils.java new file mode 100644 index 00000000..3bc2b235 --- /dev/null +++ b/src/test/java/org/qortal/test/common/AtUtils.java @@ -0,0 +1,81 @@ +package org.qortal.test.common; + +import org.ciyam.at.CompilationException; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.DeployAtTransaction; + +import java.nio.ByteBuffer; + +public class AtUtils { + + public static byte[] buildSimpleAT() { + // Pretend we use 4 values in data segment + int addrCounter = 4; + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(512); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile AT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + public static DeployAtTransaction doDeployAT(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "Test AT"; + String description = "Test AT"; + String atType = "Test"; + String tags = "TEST"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } +} diff --git a/src/test/resources/test-settings-v2-block-archive.json b/src/test/resources/test-settings-v2-block-archive.json new file mode 100644 index 00000000..612c8658 --- /dev/null +++ b/src/test/resources/test-settings-v2-block-archive.json @@ -0,0 +1,11 @@ +{ + "bitcoinNet": "TEST3", + "litecoinNet": "TEST3", + "restrictedApi": false, + "blockchainConfig": "src/test/resources/test-chain-v2.json", + "wipeUnconfirmedOnStart": false, + "testNtpOffset": 0, + "minPeers": 0, + "pruneBlockLimit": 1450, + "repositoryPath": "dbtest" +}