diff --git a/src/main/java/org/qortal/controller/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/AtStatesTrimmer.java new file mode 100644 index 00000000..d1439aae --- /dev/null +++ b/src/main/java/org/qortal/controller/AtStatesTrimmer.java @@ -0,0 +1,86 @@ +package org.qortal.controller; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.utils.NTP; + +public class AtStatesTrimmer implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(AtStatesTrimmer.class); + + private enum TrimMode { SEARCHING, TRIMMING } + private static final long TRIM_INTERVAL = 2 * 1000L; // ms + private static final int TRIM_SEARCH_SIZE = 5000; // blocks + private static final int TRIM_BATCH_SIZE = 200; // blocks + private static final int TRIM_LIMIT = 4000; // rows + + private TrimMode trimMode = TrimMode.SEARCHING; + private int trimStartHeight = 0; + + @Override + public void run() { + try (final Repository repository = RepositoryManager.getRepository()) { + while (!Controller.isStopping()) { + repository.discardChanges(); + + Thread.sleep(TRIM_INTERVAL); + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) + continue; + + long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime(); + // We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks + long chainTrimmableTimestamp = chainTip.getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime(); + + long upperTrimmableTimestamp = Math.min(currentTrimmableTimestamp, chainTrimmableTimestamp); + int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp); + + if (trimMode == TrimMode.SEARCHING) { + int trimEndHeight = Math.min(trimStartHeight + TRIM_SEARCH_SIZE, upperTrimmableHeight); + + LOGGER.debug(() -> String.format("Searching for trimmable AT states between blocks %d and %d", trimStartHeight, trimEndHeight)); + int foundStartHeight = repository.getATRepository().findFirstTrimmableStateHeight(trimStartHeight, trimEndHeight); + + if (foundStartHeight == 0) { + // No trimmable AT states found + trimStartHeight = trimEndHeight; + } else { + trimStartHeight = foundStartHeight; + trimMode = TrimMode.TRIMMING; + LOGGER.debug(() -> String.format("Found first trimmable AT state at block height %d", trimStartHeight)); + } + + // The above search will probably take enough time by itself so wait until next round + continue; + } + + int upperBatchHeight = Math.min(trimStartHeight + TRIM_BATCH_SIZE, upperTrimmableHeight); + + if (trimStartHeight >= upperBatchHeight) + continue; + + int numAtStatesTrimmed = repository.getATRepository().trimAtStates(trimStartHeight, upperBatchHeight, TRIM_LIMIT); + repository.saveChanges(); + + if (numAtStatesTrimmed > 0) { + LOGGER.debug(() -> String.format("Trimmed %d AT state%s between blocks %d and %d", + numAtStatesTrimmed, (numAtStatesTrimmed != 1 ? "s" : ""), + trimStartHeight, upperBatchHeight)); + } else { + trimStartHeight = upperBatchHeight; + } + } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to trim AT states: %s", e.getMessage())); + } catch (InterruptedException e) { + // Time to exit + } + } + +} diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 406fda79..6897751a 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -86,6 +86,7 @@ import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transaction.Transaction.ValidationResult; import org.qortal.utils.Base58; import org.qortal.utils.ByteArray; +import org.qortal.utils.DaemonThreadFactory; import org.qortal.utils.NTP; import org.qortal.utils.Triple; @@ -111,8 +112,6 @@ public class Controller extends Thread { private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms - private static final long TRIM_AT_STATES_INTERVAL = 2 * 1000L; // ms - private static final int TRIM_AT_BATCH_SIZE = 200; // blocks // To do with online accounts list private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms @@ -140,8 +139,7 @@ public class Controller extends Thread { private long repositoryBackupTimestamp = startTime; // ms private long ntpCheckTimestamp = startTime; // ms private long deleteExpiredTimestamp = startTime + DELETE_EXPIRED_INTERVAL; // ms - private long trimAtStatesTimestamp = startTime + TRIM_AT_STATES_INTERVAL; // ms - private Integer trimAtStatesStartHeight = null; + private long onlineAccountsTasksTimestamp = startTime + ONLINE_ACCOUNTS_TASKS_INTERVAL; // ms /** Whether we can mint new blocks, as reported by BlockMinter. */ @@ -417,6 +415,9 @@ public class Controller extends Thread { final long repositoryBackupInterval = Settings.getInstance().getRepositoryBackupInterval(); + Executors.newSingleThreadExecutor(new DaemonThreadFactory("AT states trimmer")).execute(new AtStatesTrimmer()); + Executors.newSingleThreadExecutor(new DaemonThreadFactory("Online sigs trimmer")).execute(new OnlineAccountsSignaturesTrimmer()); + try { while (!isStopping) { // Maybe update SysTray @@ -487,11 +488,6 @@ public class Controller extends Thread { onlineAccountsTasksTimestamp = now + ONLINE_ACCOUNTS_TASKS_INTERVAL; performOnlineAccountsTasks(); } - - if (now >= trimAtStatesTimestamp) { - trimAtStatesTimestamp = now + TRIM_AT_STATES_INTERVAL; - trimAtStates(); - } } } catch (InterruptedException e) { // Fall-through to exit @@ -1420,67 +1416,6 @@ public class Controller extends Thread { // Refresh our online accounts signatures? sendOurOnlineAccountsInfo(); - - // Trim blockchain by removing 'old' online accounts signatures - long upperMintedTimestamp = now - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime(); - trimOldOnlineAccountsSignatures(upperMintedTimestamp); - } - - private void trimOldOnlineAccountsSignatures(long upperMintedTimestamp) { - try (final Repository repository = RepositoryManager.tryRepository()) { - if (repository == null) - return; - - int numBlocksTrimmed = repository.getBlockRepository().trimOldOnlineAccountsSignatures(upperMintedTimestamp); - - if (numBlocksTrimmed > 0) - LOGGER.debug(() -> String.format("Trimmed old online accounts signatures from %d block%s", numBlocksTrimmed, (numBlocksTrimmed != 1 ? "s" : ""))); - - repository.saveChanges(); - } catch (DataException e) { - LOGGER.warn(String.format("Repository issue trying to trim old online accounts signatures: %s", e.getMessage())); - } - } - - private void trimAtStates() { - if (this.getChainTip() == null) - return; - - try (final Repository repository = RepositoryManager.tryRepository()) { - if (repository == null) - return; - - if (trimAtStatesStartHeight == null) { - trimAtStatesStartHeight = repository.getATRepository().findFirstTrimmableStateHeight(); - // The above will probably take enough time by itself - return; - } - - long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime(); - // We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks - long chainTrimmableTimestamp = this.getChainTip().getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime(); - - long upperTrimmableTimestamp = Math.min(currentTrimmableTimestamp, chainTrimmableTimestamp); - - int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp); - int upperBatchHeight = Math.min(trimAtStatesStartHeight + TRIM_AT_BATCH_SIZE, upperTrimmableHeight); - - if (trimAtStatesStartHeight >= upperBatchHeight) - return; - - int numAtStatesTrimmed = repository.getATRepository().trimAtStates(trimAtStatesStartHeight, upperBatchHeight); - repository.saveChanges(); - - if (numAtStatesTrimmed > 0) { - LOGGER.debug(() -> String.format("Trimmed %d AT state%s between blocks %d and %d", - numAtStatesTrimmed, (numAtStatesTrimmed != 1 ? "s" : ""), - trimAtStatesStartHeight, upperBatchHeight)); - } else { - trimAtStatesStartHeight = upperBatchHeight; - } - } catch (DataException e) { - LOGGER.warn(String.format("Repository issue trying to trim AT states: %s", e.getMessage())); - } } private void sendOurOnlineAccountsInfo() { diff --git a/src/main/java/org/qortal/controller/OnlineAccountsSignaturesTrimmer.java b/src/main/java/org/qortal/controller/OnlineAccountsSignaturesTrimmer.java new file mode 100644 index 00000000..139859d8 --- /dev/null +++ b/src/main/java/org/qortal/controller/OnlineAccountsSignaturesTrimmer.java @@ -0,0 +1,81 @@ +package org.qortal.controller; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.block.BlockChain; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.NTP; + +public class OnlineAccountsSignaturesTrimmer implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(OnlineAccountsSignaturesTrimmer.class); + + private enum TrimMode { SEARCHING, TRIMMING } + private static final long TRIM_INTERVAL = 2 * 1000L; // ms + private static final int TRIM_SEARCH_SIZE = 5000; // blocks + private static final int TRIM_BATCH_SIZE = 500; // blocks + + private TrimMode trimMode = TrimMode.SEARCHING; + private int trimStartHeight = 0; + + public void run() { + try (final Repository repository = RepositoryManager.getRepository()) { + while (!Controller.isStopping()) { + repository.discardChanges(); + + Thread.sleep(TRIM_INTERVAL); + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) + continue; + + // Trim blockchain by removing 'old' online accounts signatures + long upperTrimmableTimestamp = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime(); + int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp); + + if (trimMode == TrimMode.SEARCHING) { + int trimEndHeight = Math.min(trimStartHeight + TRIM_SEARCH_SIZE, upperTrimmableHeight); + + LOGGER.debug(() -> String.format("Searching for trimmable online accounts signatures between blocks %d and %d", trimStartHeight, trimEndHeight)); + int foundStartHeight = repository.getBlockRepository().findFirstTrimmableOnlineAccountsSignatureHeight(trimStartHeight, trimEndHeight); + + if (foundStartHeight == 0) { + // No trimmable online accounts signatures found + trimStartHeight = trimEndHeight; + } else { + trimStartHeight = foundStartHeight; + trimMode = TrimMode.TRIMMING; + LOGGER.debug(() -> String.format("Found first trimmable online accounts signatures at block height %d", trimStartHeight)); + } + + // The above search will probably take enough time by itself so wait until next round + continue; + } + + int upperBatchHeight = Math.min(trimStartHeight + TRIM_BATCH_SIZE, upperTrimmableHeight); + + if (trimStartHeight >= upperBatchHeight) + continue; + + int numSigsTrimmed = repository.getBlockRepository().trimOldOnlineAccountsSignatures(trimStartHeight, upperBatchHeight); + repository.saveChanges(); + + if (numSigsTrimmed > 0) { + LOGGER.debug(() -> String.format("Trimmed %d online accounts signature%s between blocks %d and %d", + numSigsTrimmed, (numSigsTrimmed != 1 ? "s" : ""), + trimStartHeight, upperBatchHeight)); + } else { + trimStartHeight = upperBatchHeight; + } + } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to trim online accounts signatures: %s", e.getMessage())); + } catch (InterruptedException e) { + // Time to exit + } + } + +} diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 9abe8c3d..509569bc 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -87,11 +87,11 @@ public interface ATRepository { */ public List getBlockATStatesAtHeight(int height) throws DataException; - /** Returns height of first trimmable AT state, or null if not found. */ - public Integer findFirstTrimmableStateHeight() throws DataException; + /** Returns height of first trimmable AT state, or 0 if not found. */ + public int findFirstTrimmableStateHeight(int minHeight, int maxHeight) throws DataException; - /** Trims non-initial full AT state data between passed heights. Returns number of trimmed rows. */ - public int trimAtStates(int minHeight, int maxHeight) throws DataException; + /** Trims full AT state data between passed heights. Returns number of trimmed rows. */ + public int trimAtStates(int minHeight, int maxHeight, int limit) throws DataException; /** * Save ATStateData into repository. diff --git a/src/main/java/org/qortal/repository/BlockRepository.java b/src/main/java/org/qortal/repository/BlockRepository.java index 4265b71f..bb2caaa1 100644 --- a/src/main/java/org/qortal/repository/BlockRepository.java +++ b/src/main/java/org/qortal/repository/BlockRepository.java @@ -143,13 +143,15 @@ public interface BlockRepository { */ public List getBlockInfos(Integer startHeight, Integer endHeight, Integer count) throws DataException; + /** Returns height of first trimmable online accounts signatures, or 0 if not found. */ + public int findFirstTrimmableOnlineAccountsSignatureHeight(int minHeight, int maxHeight) throws DataException; + /** - * Trim online accounts signatures from blocks older than passed timestamp. + * Trim online accounts signatures from blocks between passed heights. * - * @param timestamp * @return number of blocks trimmed */ - public int trimOldOnlineAccountsSignatures(long timestamp) throws DataException; + public int trimOldOnlineAccountsSignatures(int minHeight, int maxHeight) throws DataException; /** * Returns first (lowest height) block that doesn't link back to specified block. diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index 6685896c..f5e54f2a 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -400,39 +400,35 @@ public class HSQLDBATRepository implements ATRepository { } @Override - public Integer findFirstTrimmableStateHeight() throws DataException { + public int findFirstTrimmableStateHeight(int minHeight, int maxHeight) throws DataException { String sql = "SELECT MIN(height) FROM ATStates " - + "WHERE is_initial = FALSE AND state_data IS NOT NULL"; + + "WHERE state_data IS NOT NULL " + + "AND height BETWEEN ? AND ?"; - try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + try (ResultSet resultSet = this.repository.checkedExecute(sql, minHeight, maxHeight)) { if (resultSet == null) - return null; - - int height = resultSet.getInt(1); - if (height == 0 && resultSet.wasNull()) - return null; + return 0; - return height; + return resultSet.getInt(1); } catch (SQLException e) { throw new DataException("Unable to find first trimmable AT state in repository", e); } } @Override - public int trimAtStates(int minHeight, int maxHeight) throws DataException { + public int trimAtStates(int minHeight, int maxHeight, int limit) throws DataException { if (minHeight >= maxHeight) return 0; // We're often called so no need to trim all states in one go. // Limit updates to reduce CPU and memory load. String sql = "UPDATE ATStates SET state_data = NULL " - + "WHERE is_initial = FALSE " - + "AND state_data IS NOT NULL " + + "WHERE state_data IS NOT NULL " + "AND height BETWEEN ? AND ? " - + "LIMIT 4000"; + + "LIMIT ?"; try { - return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight); + return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight, limit); } catch (SQLException e) { repository.examineException(e); throw new DataException("Unable to trim AT states in repository", e); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java index 563148fd..52a6f1d0 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java @@ -462,13 +462,31 @@ public class HSQLDBBlockRepository implements BlockRepository { } @Override - public int trimOldOnlineAccountsSignatures(long timestamp) throws DataException { + public int findFirstTrimmableOnlineAccountsSignatureHeight(int minHeight, int maxHeight) throws DataException { + String sql = "SELECT MIN(height) FROM Blocks " + + "WHERE online_accounts_signatures IS NOT NULL " + + "AND height BETWEEN ? AND ?"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, minHeight, maxHeight)) { + if (resultSet == null) + return 0; + + return resultSet.getInt(1); + } catch (SQLException e) { + throw new DataException("Unable to find first trimmable online accounts signatures in repository", e); + } + } + + @Override + public int trimOldOnlineAccountsSignatures(int minHeight, int maxHeight) throws DataException { // We're often called so no need to trim all blocks in one go. // Limit updates to reduce CPU and memory load. - String sql = "UPDATE Blocks set online_accounts_signatures = NULL WHERE minted_when < ? AND online_accounts_signatures IS NOT NULL LIMIT 1440"; + String sql = "UPDATE Blocks SET online_accounts_signatures = NULL " + + "WHERE online_accounts_signatures IS NOT NULL " + + "AND height BETWEEN ? AND ?"; try { - return this.repository.executeCheckedUpdate(sql, timestamp); + return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight); } catch (SQLException e) { repository.examineException(e); throw new DataException("Unable to trim old online accounts signatures in repository", e); diff --git a/src/test/java/org/qortal/test/RepositoryTests.java b/src/test/java/org/qortal/test/RepositoryTests.java index b453ce7b..d5e70886 100644 --- a/src/test/java/org/qortal/test/RepositoryTests.java +++ b/src/test/java/org/qortal/test/RepositoryTests.java @@ -112,7 +112,7 @@ public class RepositoryTests extends Common { BlockUtils.mintBlock(repository1); // Perform database 'update', but don't commit at this stage - repository1.getBlockRepository().trimOldOnlineAccountsSignatures(System.currentTimeMillis()); + repository1.getBlockRepository().trimOldOnlineAccountsSignatures(1, 10); // Open connection 2 try (final Repository repository2 = RepositoryManager.getRepository()) {