Started work on pruning mode (top-only-sync)

Initially just deleting old and unused AT states, to get this table under control. I have had to delete them individually as the table can't handle complex queries due to its size.

Nodes in pruning mode will be unable to serve older blocks to peers.

src/main/java/org/qortal/controller/AtStatesTrimmer.java

@@ -2,6 +2,7 @@ package org.qortal.controller;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.qortal.controller.pruning.PruneManager;
 import org.qortal.data.block.BlockData;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
@@ -22,6 +23,7 @@ public class AtStatesTrimmer implements Runnable {
 
 			repository.getATRepository().prepareForAtStateTrimming();
 			repository.saveChanges();
+			PruneManager.getInstance().setBuiltLatestATStates(true);
 
 			while (!Controller.isStopping()) {
 				repository.discardChanges();

src/main/java/org/qortal/controller/Controller.java

@@ -46,6 +46,7 @@ import org.qortal.block.Block;
 import org.qortal.block.BlockChain;
 import org.qortal.block.BlockChain.BlockTimingByHeight;
 import org.qortal.controller.Synchronizer.SynchronizationResult;
+import org.qortal.controller.pruning.PruneManager;
 import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
 import org.qortal.controller.tradebot.TradeBot;
 import org.qortal.crypto.Crypto;
@@ -358,7 +359,7 @@ public class Controller extends Thread {
 		return this.savedArgs;
 	}
 
-	/* package */ static boolean isStopping() {
+	public static boolean isStopping() {
 		return isStopping;
 	}
 
@@ -1292,6 +1293,13 @@ public class Controller extends Thread {
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			BlockData blockData = repository.getBlockRepository().fromSignature(signature);
 
+			if (blockData != null) {
+				if (PruneManager.getInstance().isBlockPruned(blockData.getHeight(), repository)) {
+					// If this is a pruned block, we likely only have partial data, so best not to sent it
+					blockData = null;
+				}
+			}
+
 			if (blockData == null) {
 				// We don't have this block
 				this.stats.getBlockMessageStats.unknownBlocks.getAndIncrement();
@@ -1413,6 +1421,12 @@ public class Controller extends Thread {
 
 				BlockData blockData = repository.getBlockRepository().fromReference(parentSignature);
 
+				if (PruneManager.getInstance().isBlockPruned(blockData.getHeight(), repository)) {
+					// If this request contains a pruned block, we likely only have partial data, so best not to sent anything
+					// We always prune from the oldest first, so it's fine to just check the first block requested
+					blockData = null;
+				}
+
 				while (blockData != null && blockSummaries.size() < numberRequested) {
 					BlockSummaryData blockSummary = new BlockSummaryData(blockData);
 					blockSummaries.add(blockSummary);

src/main/java/org/qortal/controller/pruning/AtStatesPruner.java

@@ -0,0 +1,95 @@
+package org.qortal.controller.pruning;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.controller.Controller;
+import org.qortal.data.block.BlockData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.NTP;
+
+public class AtStatesPruner implements Runnable {
+
+	private static final Logger LOGGER = LogManager.getLogger(AtStatesPruner.class);
+
+	@Override
+	public void run() {
+		Thread.currentThread().setName("AT States pruner");
+
+		if (!Settings.getInstance().isPruningEnabled()) {
+			return;
+		}
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			int pruneStartHeight = repository.getATRepository().getAtPruneHeight();
+
+			// repository.getATRepository().prepareForAtStatePruning();
+			// repository.saveChanges();
+
+			while (!Controller.isStopping()) {
+				repository.discardChanges();
+
+				Thread.sleep(Settings.getInstance().getAtStatesPruneInterval());
+
+				if (PruneManager.getInstance().getBuiltLatestATStates() == false) {
+					// Wait for latest AT states table to be built first
+					// This has a dependency on the AtStatesTrimmer running,
+					// which should be okay, given that it isn't something
+					// is disabled in normal operation.
+					continue;
+				}
+
+				BlockData chainTip = Controller.getInstance().getChainTip();
+				if (chainTip == null || NTP.getTime() == null)
+					continue;
+
+				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
+				if (Controller.getInstance().isSynchronizing())
+					continue;
+
+				long currentPrunableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime();
+				// We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks
+				long chainPrunableTimestamp = chainTip.getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime();
+
+				long upperPrunableTimestamp = Math.min(currentPrunableTimestamp, chainPrunableTimestamp);
+				int upperPrunableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperPrunableTimestamp);
+
+				int upperBatchHeight = pruneStartHeight + Settings.getInstance().getAtStatesPruneBatchSize();
+				int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight);
+
+				if (pruneStartHeight >= upperPruneHeight)
+					continue;
+
+				LOGGER.debug(String.format("Pruning AT states between blocks %d and %d...", pruneStartHeight, upperPruneHeight));
+
+				int numAtStatesPruned = repository.getATRepository().pruneAtStates(pruneStartHeight, upperPruneHeight);
+				repository.saveChanges();
+
+				if (numAtStatesPruned > 0) {
+					final int finalPruneStartHeight = pruneStartHeight;
+					LOGGER.debug(() -> String.format("Pruned %d AT state%s between blocks %d and %d",
+							numAtStatesPruned, (numAtStatesPruned != 1 ? "s" : ""),
+							finalPruneStartHeight, upperPruneHeight));
+				} else {
+					// Can we move onto next batch?
+					if (upperPrunableHeight > upperBatchHeight) {
+						pruneStartHeight = upperBatchHeight;
+						repository.getATRepository().setAtPruneHeight(pruneStartHeight);
+						repository.getATRepository().prepareForAtStatePruning();
+						repository.saveChanges();
+
+						final int finalPruneStartHeight = pruneStartHeight;
+						LOGGER.debug(() -> String.format("Bumping AT state base prune height to %d", finalPruneStartHeight));
+					}
+				}
+			}
+		} catch (DataException e) {
+			LOGGER.warn(String.format("Repository issue trying to prune AT states: %s", e.getMessage()));
+		} catch (InterruptedException e) {
+			// Time to exit
+		}
+	}
+
+}

src/main/java/org/qortal/controller/pruning/PruneManager.java

@@ -0,0 +1,60 @@
+package org.qortal.controller.pruning;
+
+import org.qortal.controller.Controller;
+
+import org.qortal.data.block.BlockData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.settings.Settings;
+import org.qortal.utils.DaemonThreadFactory;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class PruneManager {
+
+    private static PruneManager instance;
+
+    private boolean pruningEnabled = Settings.getInstance().isPruningEnabled();
+    private int pruneBlockLimit = Settings.getInstance().getPruneBlockLimit();
+    private boolean builtLatestATStates = false;
+
+    private PruneManager() {
+        // Start individual pruning processes
+        ExecutorService pruneExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory());
+        pruneExecutor.execute(new AtStatesPruner());
+    }
+
+    public static synchronized PruneManager getInstance() {
+        if (instance == null)
+            instance = new PruneManager();
+
+        return instance;
+    }
+
+    public boolean isBlockPruned(int height, Repository repository) throws DataException {
+        if (!this.pruningEnabled) {
+            return false;
+        }
+
+        BlockData chainTip = Controller.getInstance().getChainTip();
+        if (chainTip == null) {
+            throw new DataException("Unable to determine chain tip when checking if a block is pruned");
+        }
+
+        final int ourLatestHeight = chainTip.getHeight();
+        final int latestUnprunedHeight = ourLatestHeight - this.pruneBlockLimit;
+
+        return (height < latestUnprunedHeight);
+    }
+
+
+    public void setBuiltLatestATStates(boolean builtLatestATStates) {
+        this.builtLatestATStates = builtLatestATStates;
+    }
+
+    public boolean getBuiltLatestATStates() {
+        return this.builtLatestATStates;
+    }
+
+}

src/main/java/org/qortal/repository/ATRepository.java

@@ -127,6 +127,23 @@ public interface ATRepository {
 	/** Trims full AT state data between passed heights. Returns number of trimmed rows. */
 	public int trimAtStates(int minHeight, int maxHeight, int limit) throws DataException;
 
+
+	/** Returns height of first prunable AT state. */
+	public int getAtPruneHeight() throws DataException;
+
+	/** Sets new base height for AT state pruning.
+	 * <p>
+	 * NOTE: performs implicit <tt>repository.saveChanges()</tt>.
+	 */
+	public void setAtPruneHeight(int pruneHeight) throws DataException;
+
+	/** Hook to allow repository to prepare/cache info for AT state pruning. */
+	public void prepareForAtStatePruning() throws DataException;
+
+	/** Prunes full AT state data between passed heights. Returns number of pruned rows. */
+	public int pruneAtStates(int minHeight, int maxHeight) throws DataException;
+
+
 	/**
 	 * Save ATStateData into repository.
 	 * <p>

src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java

@@ -8,6 +8,7 @@ import java.util.Set;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.qortal.data.account.AccountData;
 import org.qortal.data.at.ATData;
 import org.qortal.data.at.ATStateData;
 import org.qortal.repository.ATRepository;
@@ -682,6 +683,90 @@ public class HSQLDBATRepository implements ATRepository {
 		}
 	}
 
+
+	@Override
+	public int getAtPruneHeight() throws DataException {
+		String sql = "SELECT AT_prune_height FROM DatabaseInfo";
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql)) {
+			if (resultSet == null)
+				return 0;
+
+			return resultSet.getInt(1);
+		} catch (SQLException e) {
+			throw new DataException("Unable to fetch AT state prune height from repository", e);
+		}
+	}
+
+	@Override
+	public void setAtPruneHeight(int pruneHeight) throws DataException {
+		// trimHeightsLock is to prevent concurrent update on DatabaseInfo
+		// that could result in "transaction rollback: serialization failure"
+		synchronized (this.repository.trimHeightsLock) {
+			String updateSql = "UPDATE DatabaseInfo SET AT_prune_height = ?";
+
+			try {
+				this.repository.executeCheckedUpdate(updateSql, pruneHeight);
+				this.repository.saveChanges();
+			} catch (SQLException e) {
+				repository.examineException(e);
+				throw new DataException("Unable to set AT state prune height in repository", e);
+			}
+		}
+	}
+
+	@Override
+	public void prepareForAtStatePruning() throws DataException {
+		// Use LatestATStates table that was already built by AtStatesTrimmer
+		// The AtStatesPruner class checks that this process has completed first
+	}
+
+	@Override
+	public int pruneAtStates(int minHeight, int maxHeight) throws DataException {
+		int deletedCount = 0;
+
+		for (int height=minHeight; height<maxHeight; height++) {
+
+			// Get latest AT states for this height
+			List<String> atAddresses = new ArrayList<>();
+			String updateSql = "SELECT AT_address FROM LatestATStates WHERE height = ?";
+			try (ResultSet resultSet = this.repository.checkedExecute(updateSql, height)) {
+				if (resultSet != null) {
+					do {
+						String atAddress = resultSet.getString(1);
+						atAddresses.add(atAddress);
+
+					} while (resultSet.next());
+				}
+			} catch (SQLException e) {
+				throw new DataException("Unable to fetch flagged accounts from repository", e);
+			}
+
+			List<ATStateData> atStates = this.getBlockATStatesAtHeight(height);
+			for (ATStateData atState : atStates) {
+				//LOGGER.info("Found atState {} at height {}", atState.getATAddress(), atState.getHeight());
+
+				if (atAddresses.contains(atState.getATAddress())) {
+					// We don't want to delete this AT state because it is still active
+					LOGGER.info("Skipping atState {} at height {}", atState.getATAddress(), atState.getHeight());
+					continue;
+				}
+
+				// Safe to delete everything else for this height
+				try {
+					this.repository.delete("ATStates", "AT_address = ? AND height = ?",
+							atState.getATAddress(), atState.getHeight());
+					deletedCount++;
+				} catch (SQLException e) {
+					throw new DataException("Unable to delete AT state data from repository", e);
+				}
+			}
+		}
+
+		return deletedCount;
+	}
+
+
 	@Override
 	public void save(ATStateData atStateData) throws DataException {
 		// We shouldn't ever save partial ATStateData

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java

@@ -867,6 +867,10 @@ public class HSQLDBDatabaseUpdates {
 					stmt.execute("CHECKPOINT");
 					break;
 				}
+				case 35:
+					// Support for pruning
+					stmt.execute("ALTER TABLE DatabaseInfo ADD AT_prune_height INT NOT NULL DEFAULT 0");
+					break;
 
 				default:
 					// nothing to do

src/main/java/org/qortal/settings/Settings.java

@@ -106,6 +106,18 @@ public class Settings {
 	/** Max number of AT states to trim in one go. */
 	private int atStatesTrimLimit = 4000; // records
 
+	/** Whether we should prune old data to reduce database size
+	 * This prevents the node from being able to serve older blocks */
+	private boolean pruningEnabled = false;
+	/** The amount of recent blocks we should keep when pruning */
+	private int pruneBlockLimit = 1440;
+
+	/** How often to attempt AT state pruning (ms). */
+	private long atStatesPruneInterval = 3219L; // milliseconds
+	/** Block height range to scan for trimmable AT states.<br>
+	 * This has a significant effect on execution time. */
+	private int atStatesPruneBatchSize = 10; // blocks
+
 	/** How often to attempt online accounts signatures trimming (ms). */
 	private long onlineSignaturesTrimInterval = 9876L; // milliseconds
 	/** Block height range to scan for trimmable online accounts signatures.<br>
@@ -528,6 +540,22 @@ public class Settings {
 		return this.atStatesTrimLimit;
 	}
 
+	public boolean isPruningEnabled() {
+		return this.pruningEnabled;
+	}
+
+	public int getPruneBlockLimit() {
+		return this.pruneBlockLimit;
+	}
+
+	public long getAtStatesPruneInterval() {
+		return this.atStatesPruneInterval;
+	}
+
+	public int getAtStatesPruneBatchSize() {
+		return this.atStatesPruneBatchSize;
+	}
+
 	public long getOnlineSignaturesTrimInterval() {
 		return this.onlineSignaturesTrimInterval;
 	}