Aggressively trim old AT state data and online accounts signatures.

Two new classes/threads made to quickly find first trimmable row
then repeatedly trim rows in small batches after that.

src/main/java/org/qortal/controller/AtStatesTrimmer.java

@@ -0,0 +1,86 @@
+package org.qortal.controller;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.data.block.BlockData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.NTP;
+
+public class AtStatesTrimmer implements Runnable {
+
+	private static final Logger LOGGER = LogManager.getLogger(AtStatesTrimmer.class);
+
+	private enum TrimMode { SEARCHING, TRIMMING }
+	private static final long TRIM_INTERVAL = 2 * 1000L; // ms
+	private static final int TRIM_SEARCH_SIZE = 5000; // blocks
+	private static final int TRIM_BATCH_SIZE = 200; // blocks
+	private static final int TRIM_LIMIT = 4000; // rows
+
+	private TrimMode trimMode = TrimMode.SEARCHING;
+	private int trimStartHeight = 0;
+
+	@Override
+	public void run() {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			while (!Controller.isStopping()) {
+				repository.discardChanges();
+
+				Thread.sleep(TRIM_INTERVAL);
+
+				BlockData chainTip = Controller.getInstance().getChainTip();
+				if (chainTip == null || NTP.getTime() == null)
+					continue;
+
+				long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime();
+				// We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks
+				long chainTrimmableTimestamp = chainTip.getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime();
+
+				long upperTrimmableTimestamp = Math.min(currentTrimmableTimestamp, chainTrimmableTimestamp);
+				int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp);
+
+				if (trimMode == TrimMode.SEARCHING) {
+					int trimEndHeight = Math.min(trimStartHeight + TRIM_SEARCH_SIZE, upperTrimmableHeight);
+
+					LOGGER.debug(() -> String.format("Searching for trimmable AT states between blocks %d and %d", trimStartHeight, trimEndHeight));
+					int foundStartHeight = repository.getATRepository().findFirstTrimmableStateHeight(trimStartHeight, trimEndHeight);
+
+					if (foundStartHeight == 0) {
+						// No trimmable AT states found
+						trimStartHeight = trimEndHeight;
+					} else {
+						trimStartHeight = foundStartHeight;
+						trimMode = TrimMode.TRIMMING;
+						LOGGER.debug(() -> String.format("Found first trimmable AT state at block height %d", trimStartHeight));
+					}
+
+					// The above search will probably take enough time by itself so wait until next round 
+					continue;
+				}
+
+				int upperBatchHeight = Math.min(trimStartHeight + TRIM_BATCH_SIZE, upperTrimmableHeight);
+
+				if (trimStartHeight >= upperBatchHeight)
+					continue;
+
+				int numAtStatesTrimmed = repository.getATRepository().trimAtStates(trimStartHeight, upperBatchHeight, TRIM_LIMIT);
+				repository.saveChanges();
+
+				if (numAtStatesTrimmed > 0) {
+					LOGGER.debug(() -> String.format("Trimmed %d AT state%s between blocks %d and %d",
+							numAtStatesTrimmed, (numAtStatesTrimmed != 1 ? "s" : ""),
+							trimStartHeight, upperBatchHeight));
+				} else {
+					trimStartHeight = upperBatchHeight;
+				}
+			}
+		} catch (DataException e) {
+			LOGGER.warn(String.format("Repository issue trying to trim AT states: %s", e.getMessage()));
+		} catch (InterruptedException e) {
+			// Time to exit
+		}
+	}
+
+}

src/main/java/org/qortal/controller/Controller.java

@@ -86,6 +86,7 @@ import org.qortal.transaction.Transaction.TransactionType;
 import org.qortal.transaction.Transaction.ValidationResult;
 import org.qortal.utils.Base58;
 import org.qortal.utils.ByteArray;
+import org.qortal.utils.DaemonThreadFactory;
 import org.qortal.utils.NTP;
 import org.qortal.utils.Triple;
 
@@ -111,8 +112,6 @@ public class Controller extends Thread {
 	private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms
 	private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms
 	private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms
-	private static final long TRIM_AT_STATES_INTERVAL = 2 * 1000L; // ms
-	private static final int TRIM_AT_BATCH_SIZE = 200; // blocks
 
 	// To do with online accounts list
 	private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms
@@ -140,8 +139,7 @@ public class Controller extends Thread {
 	private long repositoryBackupTimestamp = startTime; // ms
 	private long ntpCheckTimestamp = startTime; // ms
 	private long deleteExpiredTimestamp = startTime + DELETE_EXPIRED_INTERVAL; // ms
-	private long trimAtStatesTimestamp = startTime + TRIM_AT_STATES_INTERVAL; // ms
-	private Integer trimAtStatesStartHeight = null;
+
 	private long onlineAccountsTasksTimestamp = startTime + ONLINE_ACCOUNTS_TASKS_INTERVAL; // ms
 
 	/** Whether we can mint new blocks, as reported by BlockMinter. */
@@ -417,6 +415,9 @@ public class Controller extends Thread {
 
 		final long repositoryBackupInterval = Settings.getInstance().getRepositoryBackupInterval();
 
+		Executors.newSingleThreadExecutor(new DaemonThreadFactory("AT states trimmer")).execute(new AtStatesTrimmer());
+		Executors.newSingleThreadExecutor(new DaemonThreadFactory("Online sigs trimmer")).execute(new OnlineAccountsSignaturesTrimmer());
+
 		try {
 			while (!isStopping) {
 				// Maybe update SysTray
@@ -487,11 +488,6 @@ public class Controller extends Thread {
 					onlineAccountsTasksTimestamp = now + ONLINE_ACCOUNTS_TASKS_INTERVAL;
 					performOnlineAccountsTasks();
 				}
-
-				if (now >= trimAtStatesTimestamp) {
-					trimAtStatesTimestamp = now + TRIM_AT_STATES_INTERVAL;
-					trimAtStates();
-				}
 			}
 		} catch (InterruptedException e) {
 			// Fall-through to exit
@@ -1420,67 +1416,6 @@ public class Controller extends Thread {
 
 		// Refresh our online accounts signatures?
 		sendOurOnlineAccountsInfo();
-
-		// Trim blockchain by removing 'old' online accounts signatures
-		long upperMintedTimestamp = now - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime();
-		trimOldOnlineAccountsSignatures(upperMintedTimestamp);
-	}
-
-	private void trimOldOnlineAccountsSignatures(long upperMintedTimestamp) {
-		try (final Repository repository = RepositoryManager.tryRepository()) {
-			if (repository == null)
-				return;
-
-			int numBlocksTrimmed = repository.getBlockRepository().trimOldOnlineAccountsSignatures(upperMintedTimestamp);
-
-			if (numBlocksTrimmed > 0)
-				LOGGER.debug(() -> String.format("Trimmed old online accounts signatures from %d block%s", numBlocksTrimmed, (numBlocksTrimmed != 1 ? "s" : "")));
-
-			repository.saveChanges();
-		} catch (DataException e) {
-			LOGGER.warn(String.format("Repository issue trying to trim old online accounts signatures: %s", e.getMessage()));
-		}
-	}
-
-	private void trimAtStates() {
-		if (this.getChainTip() == null)
-			return;
-
-		try (final Repository repository = RepositoryManager.tryRepository()) {
-			if (repository == null)
-				return;
-
-			if (trimAtStatesStartHeight == null) {
-				trimAtStatesStartHeight = repository.getATRepository().findFirstTrimmableStateHeight();
-				// The above will probably take enough time by itself
-				return;
-			}
-
-			long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime();
-			// We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks
-			long chainTrimmableTimestamp = this.getChainTip().getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime();
-
-			long upperTrimmableTimestamp = Math.min(currentTrimmableTimestamp, chainTrimmableTimestamp);
-
-			int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp);
-			int upperBatchHeight = Math.min(trimAtStatesStartHeight + TRIM_AT_BATCH_SIZE, upperTrimmableHeight);
-
-			if (trimAtStatesStartHeight >= upperBatchHeight)
-				return;
-
-			int numAtStatesTrimmed = repository.getATRepository().trimAtStates(trimAtStatesStartHeight, upperBatchHeight);
-			repository.saveChanges();
-
-			if (numAtStatesTrimmed > 0) {
-				LOGGER.debug(() -> String.format("Trimmed %d AT state%s between blocks %d and %d",
-						numAtStatesTrimmed, (numAtStatesTrimmed != 1 ? "s" : ""),
-						trimAtStatesStartHeight, upperBatchHeight));
-			} else {
-				trimAtStatesStartHeight = upperBatchHeight;
-			}
-		} catch (DataException e) {
-			LOGGER.warn(String.format("Repository issue trying to trim AT states: %s", e.getMessage()));
-		}
 	}
 
 	private void sendOurOnlineAccountsInfo() {

src/main/java/org/qortal/controller/OnlineAccountsSignaturesTrimmer.java

@@ -0,0 +1,81 @@
+package org.qortal.controller;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.block.BlockChain;
+import org.qortal.data.block.BlockData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.utils.NTP;
+
+public class OnlineAccountsSignaturesTrimmer implements Runnable {
+
+	private static final Logger LOGGER = LogManager.getLogger(OnlineAccountsSignaturesTrimmer.class);
+
+	private enum TrimMode { SEARCHING, TRIMMING }
+	private static final long TRIM_INTERVAL = 2 * 1000L; // ms
+	private static final int TRIM_SEARCH_SIZE = 5000; // blocks
+	private static final int TRIM_BATCH_SIZE = 500; // blocks
+
+	private TrimMode trimMode = TrimMode.SEARCHING;
+	private int trimStartHeight = 0;
+
+	public void run() {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			while (!Controller.isStopping()) {
+				repository.discardChanges();
+
+				Thread.sleep(TRIM_INTERVAL);
+
+				BlockData chainTip = Controller.getInstance().getChainTip();
+				if (chainTip == null || NTP.getTime() == null)
+					continue;
+
+				// Trim blockchain by removing 'old' online accounts signatures
+				long upperTrimmableTimestamp = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime();
+				int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp);
+
+				if (trimMode == TrimMode.SEARCHING) {
+					int trimEndHeight = Math.min(trimStartHeight + TRIM_SEARCH_SIZE, upperTrimmableHeight);
+
+					LOGGER.debug(() -> String.format("Searching for trimmable online accounts signatures between blocks %d and %d", trimStartHeight, trimEndHeight));
+					int foundStartHeight = repository.getBlockRepository().findFirstTrimmableOnlineAccountsSignatureHeight(trimStartHeight, trimEndHeight);
+
+					if (foundStartHeight == 0) {
+						// No trimmable online accounts signatures found
+						trimStartHeight = trimEndHeight;
+					} else {
+						trimStartHeight = foundStartHeight;
+						trimMode = TrimMode.TRIMMING;
+						LOGGER.debug(() -> String.format("Found first trimmable online accounts signatures at block height %d", trimStartHeight));
+					}
+
+					// The above search will probably take enough time by itself so wait until next round 
+					continue;
+				}
+
+				int upperBatchHeight = Math.min(trimStartHeight + TRIM_BATCH_SIZE, upperTrimmableHeight);
+
+				if (trimStartHeight >= upperBatchHeight)
+					continue;
+
+				int numSigsTrimmed = repository.getBlockRepository().trimOldOnlineAccountsSignatures(trimStartHeight, upperBatchHeight);
+				repository.saveChanges();
+
+				if (numSigsTrimmed > 0) {
+					LOGGER.debug(() -> String.format("Trimmed %d online accounts signature%s between blocks %d and %d",
+							numSigsTrimmed, (numSigsTrimmed != 1 ? "s" : ""),
+							trimStartHeight, upperBatchHeight));
+				} else {
+					trimStartHeight = upperBatchHeight;
+				}
+			}
+		} catch (DataException e) {
+			LOGGER.warn(String.format("Repository issue trying to trim online accounts signatures: %s", e.getMessage()));
+		} catch (InterruptedException e) {
+			// Time to exit
+		}
+	}
+
+}

src/main/java/org/qortal/repository/ATRepository.java

@@ -87,11 +87,11 @@ public interface ATRepository {
 	 */
 	public List<ATStateData> getBlockATStatesAtHeight(int height) throws DataException;
 
-	/** Returns height of first trimmable AT state, or null if not found. */
-	public Integer findFirstTrimmableStateHeight() throws DataException;
+	/** Returns height of first trimmable AT state, or 0 if not found. */
+	public int findFirstTrimmableStateHeight(int minHeight, int maxHeight) throws DataException;
 
-	/** Trims non-initial full AT state data between passed heights. Returns number of trimmed rows. */
-	public int trimAtStates(int minHeight, int maxHeight) throws DataException;
+	/** Trims full AT state data between passed heights. Returns number of trimmed rows. */
+	public int trimAtStates(int minHeight, int maxHeight, int limit) throws DataException;
 
 	/**
 	 * Save ATStateData into repository.

src/main/java/org/qortal/repository/BlockRepository.java

@@ -143,13 +143,15 @@ public interface BlockRepository {
 	 */
 	public List<BlockInfo> getBlockInfos(Integer startHeight, Integer endHeight, Integer count) throws DataException;
 
+	/** Returns height of first trimmable online accounts signatures, or 0 if not found. */
+	public int findFirstTrimmableOnlineAccountsSignatureHeight(int minHeight, int maxHeight) throws DataException;
+
 	/**
-	 * Trim online accounts signatures from blocks older than passed timestamp.
+	 * Trim online accounts signatures from blocks between passed heights.
 	 * 
-	 * @param timestamp
 	 * @return number of blocks trimmed
 	 */
-	public int trimOldOnlineAccountsSignatures(long timestamp) throws DataException;
+	public int trimOldOnlineAccountsSignatures(int minHeight, int maxHeight) throws DataException;
 
 	/**
 	 * Returns first (lowest height) block that doesn't link back to specified block.

src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java

@@ -400,39 +400,35 @@ public class HSQLDBATRepository implements ATRepository {
 	}
 
 	@Override
-	public Integer findFirstTrimmableStateHeight() throws DataException {
+	public int findFirstTrimmableStateHeight(int minHeight, int maxHeight) throws DataException {
 		String sql = "SELECT MIN(height) FROM ATStates "
-				+ "WHERE is_initial = FALSE AND state_data IS NOT NULL";
+				+ "WHERE state_data IS NOT NULL "
+				+ "AND height BETWEEN ? AND ?";
 
-		try (ResultSet resultSet = this.repository.checkedExecute(sql)) {
+		try (ResultSet resultSet = this.repository.checkedExecute(sql, minHeight, maxHeight)) {
 			if (resultSet == null)
-				return null;
+				return 0;
 
-			int height = resultSet.getInt(1);
-			if (height == 0 && resultSet.wasNull())
-				return null;
-
-			return height;
+			return resultSet.getInt(1);
 		} catch (SQLException e) {
 			throw new DataException("Unable to find first trimmable AT state in repository", e);
 		}
 	}
 
 	@Override
-	public int trimAtStates(int minHeight, int maxHeight) throws DataException {
+	public int trimAtStates(int minHeight, int maxHeight, int limit) throws DataException {
 		if (minHeight >= maxHeight)
 			return 0;
 
 		// We're often called so no need to trim all states in one go.
 		// Limit updates to reduce CPU and memory load.
 		String sql = "UPDATE ATStates SET state_data = NULL "
-				+ "WHERE is_initial = FALSE "
-				+ "AND state_data IS NOT NULL "
+				+ "WHERE state_data IS NOT NULL "
 				+ "AND height BETWEEN ? AND ? "
-				+ "LIMIT 4000";
+				+ "LIMIT ?";
 
 		try {
-			return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight);
+			return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight, limit);
 		} catch (SQLException e) {
 			repository.examineException(e);
 			throw new DataException("Unable to trim AT states in repository", e);

src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java

@@ -462,13 +462,31 @@ public class HSQLDBBlockRepository implements BlockRepository {
 	}
 
 	@Override
-	public int trimOldOnlineAccountsSignatures(long timestamp) throws DataException {
+	public int findFirstTrimmableOnlineAccountsSignatureHeight(int minHeight, int maxHeight) throws DataException {
+		String sql = "SELECT MIN(height) FROM Blocks "
+				+ "WHERE online_accounts_signatures IS NOT NULL "
+				+ "AND height BETWEEN ? AND ?";
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql, minHeight, maxHeight)) {
+			if (resultSet == null)
+				return 0;
+
+			return resultSet.getInt(1);
+		} catch (SQLException e) {
+			throw new DataException("Unable to find first trimmable online accounts signatures in repository", e);
+		}
+	}
+
+	@Override
+	public int trimOldOnlineAccountsSignatures(int minHeight, int maxHeight) throws DataException {
 		// We're often called so no need to trim all blocks in one go.
 		// Limit updates to reduce CPU and memory load.
-		String sql = "UPDATE Blocks set online_accounts_signatures = NULL WHERE minted_when < ? AND online_accounts_signatures IS NOT NULL LIMIT 1440";
+		String sql = "UPDATE Blocks SET online_accounts_signatures = NULL "
+				+ "WHERE online_accounts_signatures IS NOT NULL "
+				+ "AND height BETWEEN ? AND ?";
 
 		try {
-			return this.repository.executeCheckedUpdate(sql, timestamp);
+			return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight);
 		} catch (SQLException e) {
 			repository.examineException(e);
 			throw new DataException("Unable to trim old online accounts signatures in repository", e);

src/test/java/org/qortal/test/RepositoryTests.java

@@ -112,7 +112,7 @@ public class RepositoryTests extends Common {
 				BlockUtils.mintBlock(repository1);
 
 			// Perform database 'update', but don't commit at this stage
-			repository1.getBlockRepository().trimOldOnlineAccountsSignatures(System.currentTimeMillis());
+			repository1.getBlockRepository().trimOldOnlineAccountsSignatures(1, 10);
 
 			// Open connection 2
 			try (final Repository repository2 = RepositoryManager.getRepository()) {