Reduce DB storage of AT states

Drop created_when column from ATStates as it never changes
and can be fetched from ATs table.
This takes about 50s on a fast machine.

Correspondingly rebuild height-based index on ATStates.
This takes about 3 minutes on a fast machine.

Modify AT-related repository methods and callers.

Aggressively remove 'old' (> 2 weeks) actual AT
state binary data, leaving only the hash in DB
(for syncing purposes). Seems to keep up with
syncing from another node on localhost.

src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java

@@ -256,7 +256,7 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener {
 
 		if (crossChainTradeData.mode == BTCACCT.Mode.OFFERING)
 			// We want when trade was created, not when it was last updated
-			atStateTimestamp = atState.getCreation();
+			atStateTimestamp = crossChainTradeData.creationTimestamp;
 		else
 			atStateTimestamp = timestamp != null ? timestamp : repository.getBlockRepository().getTimestampFromHeight(atState.getHeight());
 

src/main/java/org/qortal/at/AT.java

@@ -61,7 +61,7 @@ public class AT {
 		byte[] stateData = machineState.toBytes();
 		byte[] stateHash = Crypto.digest(stateData);
 
-		this.atStateData = new ATStateData(atAddress, height, creation, stateData, stateHash, 0L, true);
+		this.atStateData = new ATStateData(atAddress, height, stateData, stateHash, 0L, true);
 	}
 
 	// Getters / setters
@@ -107,12 +107,11 @@ public class AT {
 			throw new DataException(String.format("Uncaught exception while running AT '%s'", atAddress), e);
 		}
 
-		long creation = this.atData.getCreation();
 		byte[] stateData = state.toBytes();
 		byte[] stateHash = Crypto.digest(stateData);
 		long atFees = api.calcFinalFees(state);
 
-		this.atStateData = new ATStateData(atAddress, blockHeight, creation, stateData, stateHash, atFees, false);
+		this.atStateData = new ATStateData(atAddress, blockHeight, stateData, stateHash, atFees, false);
 
 		return api.getTransactions();
 	}

src/main/java/org/qortal/controller/Controller.java

@@ -111,6 +111,8 @@ public class Controller extends Thread {
 	private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms
 	private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms
 	private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms
+	private static final long TRIM_AT_STATES_INTERVAL = 2 * 1000L; // ms
+	private static final int TRIM_AT_BATCH_SIZE = 200; // blocks
 
 	// To do with online accounts list
 	private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms
@@ -138,6 +140,8 @@ public class Controller extends Thread {
 	private long repositoryBackupTimestamp = startTime; // ms
 	private long ntpCheckTimestamp = startTime; // ms
 	private long deleteExpiredTimestamp = startTime + DELETE_EXPIRED_INTERVAL; // ms
+	private long trimAtStatesTimestamp = startTime + TRIM_AT_STATES_INTERVAL; // ms
+	private Integer trimAtStatesStartHeight = null;
 	private long onlineAccountsTasksTimestamp = startTime + ONLINE_ACCOUNTS_TASKS_INTERVAL; // ms
 
 	/** Whether we can mint new blocks, as reported by BlockMinter. */
@@ -483,6 +487,11 @@ public class Controller extends Thread {
 					onlineAccountsTasksTimestamp = now + ONLINE_ACCOUNTS_TASKS_INTERVAL;
 					performOnlineAccountsTasks();
 				}
+
+				if (now >= trimAtStatesTimestamp) {
+					trimAtStatesTimestamp = now + TRIM_AT_STATES_INTERVAL;
+					trimAtStates();
+				}
 			}
 		} catch (InterruptedException e) {
 			// Fall-through to exit
@@ -1427,6 +1436,47 @@ public class Controller extends Thread {
 		}
 	}
 
+	private void trimAtStates() {
+		if (this.getChainTip() == null)
+			return;
+
+		try (final Repository repository = RepositoryManager.tryRepository()) {
+			if (repository == null)
+				return;
+
+			if (trimAtStatesStartHeight == null) {
+				trimAtStatesStartHeight = repository.getATRepository().findFirstTrimmableStateHeight();
+				// The above will probably take enough time by itself
+				return;
+			}
+
+			long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime();
+			// We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks
+			long chainTrimmableTimestamp = this.getChainTip().getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime();
+
+			long upperTrimmableTimestamp = Math.min(currentTrimmableTimestamp, chainTrimmableTimestamp);
+
+			int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp);
+			int upperBatchHeight = Math.min(trimAtStatesStartHeight + TRIM_AT_BATCH_SIZE, upperTrimmableHeight);
+
+			if (trimAtStatesStartHeight >= upperBatchHeight)
+				return;
+
+			int numAtStatesTrimmed = repository.getATRepository().trimAtStates(trimAtStatesStartHeight, upperBatchHeight);
+			repository.saveChanges();
+
+			if (numAtStatesTrimmed > 0) {
+				LOGGER.debug(() -> String.format("Trimmed %d AT state%s between blocks %d and %d",
+						numAtStatesTrimmed, (numAtStatesTrimmed != 1 ? "s" : ""),
+						trimAtStatesStartHeight, upperBatchHeight));
+			} else {
+				trimAtStatesStartHeight = upperBatchHeight;
+			}
+		} catch (DataException e) {
+			LOGGER.warn(String.format("Repository issue trying to trim AT states: %s", e.getMessage()));
+		}
+	}
+
 	private void sendOurOnlineAccountsInfo() {
 		final Long now = NTP.getTime();
 		if (now == null)

src/main/java/org/qortal/crosschain/BTCACCT.java

@@ -611,7 +611,7 @@ public class BTCACCT {
 	 */
 	public static CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException {
 		ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atStateData);
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
@@ -622,8 +622,8 @@ public class BTCACCT {
 	 * @throws DataException
 	 */
 	public static CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException {
-		byte[] creatorPublicKey = repository.getATRepository().getCreatorPublicKey(atStateData.getATAddress());
-		return populateTradeData(repository, creatorPublicKey, atStateData);
+		ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress());
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
@@ -633,7 +633,7 @@ public class BTCACCT {
 	 * @param atAddress
 	 * @throws DataException
 	 */
-	public static CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, ATStateData atStateData) throws DataException {
+	public static CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException {
 		byte[] addressBytes = new byte[25]; // for general use
 		String atAddress = atStateData.getATAddress();
 
@@ -641,7 +641,7 @@ public class BTCACCT {
 
 		tradeData.qortalAtAddress = atAddress;
 		tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey);
-		tradeData.creationTimestamp = atStateData.getCreation();
+		tradeData.creationTimestamp = creationTimestamp;
 
 		Account atAccount = new Account(repository, atAddress);
 		tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);

src/main/java/org/qortal/data/at/ATStateData.java

@@ -5,7 +5,6 @@ public class ATStateData {
 	// Properties
 	private String ATAddress;
 	private Integer height;
-	private Long creation;
 	private byte[] stateData;
 	private byte[] stateHash;
 	private Long fees;
@@ -14,10 +13,9 @@ public class ATStateData {
 	// Constructors
 
 	/** Create new ATStateData */
-	public ATStateData(String ATAddress, Integer height, Long creation, byte[] stateData, byte[] stateHash, Long fees, boolean isInitial) {
+	public ATStateData(String ATAddress, Integer height, byte[] stateData, byte[] stateHash, Long fees, boolean isInitial) {
 		this.ATAddress = ATAddress;
 		this.height = height;
-		this.creation = creation;
 		this.stateData = stateData;
 		this.stateHash = stateHash;
 		this.fees = fees;
@@ -26,21 +24,21 @@ public class ATStateData {
 
 	/** For recreating per-block ATStateData from repository where not all info is needed */
 	public ATStateData(String ATAddress, int height, byte[] stateHash, Long fees, boolean isInitial) {
-		this(ATAddress, height, null, null, stateHash, fees, isInitial);
+		this(ATAddress, height, null, stateHash, fees, isInitial);
 	}
 
 	/** For creating ATStateData from serialized bytes when we don't have all the info */
 	public ATStateData(String ATAddress, byte[] stateHash) {
 		// This won't ever be initial AT state from deployment as that's never serialized over the network,
 		// but generated when the DeployAtTransaction is processed locally.
-		this(ATAddress, null, null, null, stateHash, null, false);
+		this(ATAddress, null, null, stateHash, null, false);
 	}
 
 	/** For creating ATStateData from serialized bytes when we don't have all the info */
 	public ATStateData(String ATAddress, byte[] stateHash, Long fees) {
 		// This won't ever be initial AT state from deployment as that's never serialized over the network,
 		// but generated when the DeployAtTransaction is processed locally.
-		this(ATAddress, null, null, null, stateHash, fees, false);
+		this(ATAddress, null, null, stateHash, fees, false);
 	}
 
 	// Getters / setters
@@ -58,10 +56,6 @@ public class ATStateData {
 		this.height = height;
 	}
 
-	public Long getCreation() {
-		return this.creation;
-	}
-
 	public byte[] getStateData() {
 		return this.stateData;
 	}

src/main/java/org/qortal/repository/ATRepository.java

@@ -87,6 +87,12 @@ public interface ATRepository {
 	 */
 	public List<ATStateData> getBlockATStatesAtHeight(int height) throws DataException;
 
+	/** Returns height of first trimmable AT state, or null if not found. */
+	public Integer findFirstTrimmableStateHeight() throws DataException;
+
+	/** Trims non-initial full AT state data between passed heights. Returns number of trimmed rows. */
+	public int trimAtStates(int minHeight, int maxHeight) throws DataException;
+
 	/**
 	 * Save ATStateData into repository.
 	 * <p>

src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java

@@ -248,7 +248,7 @@ public class HSQLDBATRepository implements ATRepository {
 
 	@Override
 	public ATStateData getATStateAtHeight(String atAddress, int height) throws DataException {
-		String sql = "SELECT created_when, state_data, state_hash, fees, is_initial "
+		String sql = "SELECT state_data, state_hash, fees, is_initial "
 				+ "FROM ATStates "
 				+ "WHERE AT_address = ? AND height = ? "
 				+ "LIMIT 1";
@@ -257,13 +257,12 @@ public class HSQLDBATRepository implements ATRepository {
 			if (resultSet == null)
 				return null;
 
-			long created = resultSet.getLong(1);
-			byte[] stateData = resultSet.getBytes(2); // Actually BLOB
-			byte[] stateHash = resultSet.getBytes(3);
-			long fees = resultSet.getLong(4);
-			boolean isInitial = resultSet.getBoolean(5);
+			byte[] stateData = resultSet.getBytes(1); // Actually BLOB
+			byte[] stateHash = resultSet.getBytes(2);
+			long fees = resultSet.getLong(3);
+			boolean isInitial = resultSet.getBoolean(4);
 
-			return new ATStateData(atAddress, height, created, stateData, stateHash, fees, isInitial);
+			return new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial);
 		} catch (SQLException e) {
 			throw new DataException("Unable to fetch AT state from repository", e);
 		}
@@ -271,7 +270,7 @@ public class HSQLDBATRepository implements ATRepository {
 
 	@Override
 	public ATStateData getLatestATState(String atAddress) throws DataException {
-		String sql = "SELECT height, created_when, state_data, state_hash, fees, is_initial "
+		String sql = "SELECT height, state_data, state_hash, fees, is_initial "
 				+ "FROM ATStates "
 				+ "WHERE AT_address = ? "
 				// AT_address then height so the compound primary key is used as an index
@@ -284,13 +283,12 @@ public class HSQLDBATRepository implements ATRepository {
 				return null;
 
 			int height = resultSet.getInt(1);
-			long created = resultSet.getLong(2);
-			byte[] stateData = resultSet.getBytes(3); // Actually BLOB
-			byte[] stateHash = resultSet.getBytes(4);
-			long fees = resultSet.getLong(5);
-			boolean isInitial = resultSet.getBoolean(6);
+			byte[] stateData = resultSet.getBytes(2); // Actually BLOB
+			byte[] stateHash = resultSet.getBytes(3);
+			long fees = resultSet.getLong(4);
+			boolean isInitial = resultSet.getBoolean(5);
 
-			return new ATStateData(atAddress, height, created, stateData, stateHash, fees, isInitial);
+			return new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial);
 		} catch (SQLException e) {
 			throw new DataException("Unable to fetch latest AT state from repository", e);
 		}
@@ -303,10 +301,10 @@ public class HSQLDBATRepository implements ATRepository {
 		StringBuilder sql = new StringBuilder(1024);
 		List<Object> bindParams = new ArrayList<>();
 
-		sql.append("SELECT AT_address, height, created_when, state_data, state_hash, fees, is_initial "
+		sql.append("SELECT AT_address, height, state_data, state_hash, fees, is_initial "
 				+ "FROM ATs "
 				+ "CROSS JOIN LATERAL("
-					+ "SELECT height, created_when, state_data, state_hash, fees, is_initial "
+					+ "SELECT height, state_data, state_hash, fees, is_initial "
 					+ "FROM ATStates "
 					+ "WHERE ATStates.AT_address = ATs.AT_address ");
 
@@ -354,13 +352,12 @@ public class HSQLDBATRepository implements ATRepository {
 			do {
 				String atAddress = resultSet.getString(1);
 				int height = resultSet.getInt(2);
-				long created = resultSet.getLong(3);
-				byte[] stateData = resultSet.getBytes(4); // Actually BLOB
-				byte[] stateHash = resultSet.getBytes(5);
-				long fees = resultSet.getLong(6);
-				boolean isInitial = resultSet.getBoolean(7);
+				byte[] stateData = resultSet.getBytes(3); // Actually BLOB
+				byte[] stateHash = resultSet.getBytes(4);
+				long fees = resultSet.getLong(5);
+				boolean isInitial = resultSet.getBoolean(6);
 
-				ATStateData atStateData = new ATStateData(atAddress, height, created, stateData, stateHash, fees, isInitial);
+				ATStateData atStateData = new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial);
 
 				atStates.add(atStateData);
 			} while (resultSet.next());
@@ -375,6 +372,7 @@ public class HSQLDBATRepository implements ATRepository {
 	public List<ATStateData> getBlockATStatesAtHeight(int height) throws DataException {
 		String sql = "SELECT AT_address, state_hash, fees, is_initial "
 				+ "FROM ATStates "
+				+ "LEFT OUTER JOIN ATs USING (AT_address) "
 				+ "WHERE height = ? "
 				+ "ORDER BY created_when ASC";
 
@@ -401,18 +399,57 @@ public class HSQLDBATRepository implements ATRepository {
 		return atStates;
 	}
 
+	@Override
+	public Integer findFirstTrimmableStateHeight() throws DataException {
+		String sql = "SELECT MIN(height) FROM ATStates "
+				+ "WHERE is_initial = FALSE AND state_data IS NOT NULL";
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql)) {
+			if (resultSet == null)
+				return null;
+
+			int height = resultSet.getInt(1);
+			if (height == 0 && resultSet.wasNull())
+				return null;
+
+			return height;
+		} catch (SQLException e) {
+			throw new DataException("Unable to find first trimmable AT state in repository", e);
+		}
+	}
+
+	@Override
+	public int trimAtStates(int minHeight, int maxHeight) throws DataException {
+		if (minHeight >= maxHeight)
+			return 0;
+
+		// We're often called so no need to trim all states in one go.
+		// Limit updates to reduce CPU and memory load.
+		String sql = "UPDATE ATStates SET state_data = NULL "
+				+ "WHERE is_initial = FALSE "
+				+ "AND state_data IS NOT NULL "
+				+ "AND height BETWEEN ? AND ? "
+				+ "LIMIT 4000";
+
+		try {
+			return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight);
+		} catch (SQLException e) {
+			repository.examineException(e);
+			throw new DataException("Unable to trim AT states in repository", e);
+		}
+	}
+
 	@Override
 	public void save(ATStateData atStateData) throws DataException {
 		// We shouldn't ever save partial ATStateData
-		if (atStateData.getCreation() == null || atStateData.getStateHash() == null || atStateData.getHeight() == null)
+		if (atStateData.getStateHash() == null || atStateData.getHeight() == null)
 			throw new IllegalArgumentException("Refusing to save partial AT state into repository!");
 
 		HSQLDBSaver saveHelper = new HSQLDBSaver("ATStates");
 
 		saveHelper.bind("AT_address", atStateData.getATAddress()).bind("height", atStateData.getHeight())
-				.bind("created_when", atStateData.getCreation()).bind("state_data", atStateData.getStateData())
-				.bind("state_hash", atStateData.getStateHash()).bind("fees", atStateData.getFees())
-				.bind("is_initial", atStateData.isInitial());
+				.bind("state_data", atStateData.getStateData()).bind("state_hash", atStateData.getStateHash())
+				.bind("fees", atStateData.getFees()).bind("is_initial", atStateData.isInitial());
 
 		try {
 			saveHelper.execute(this.repository);

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java

@@ -212,6 +212,8 @@ public class HSQLDBDatabaseUpdates {
 							+ "PRIMARY KEY (account))");
 					// For looking up an account by public key
 					stmt.execute("CREATE INDEX AccountPublicKeyIndex on Accounts (public_key)");
+					// Use a separate table space as this table will be very large.
+					stmt.execute("SET TABLE Accounts NEW SPACE");
 
 					// Account balances
 					stmt.execute("CREATE TABLE AccountBalances (account QortalAddress, asset_id AssetID, balance QortalAmount NOT NULL, "
@@ -220,6 +222,8 @@ public class HSQLDBDatabaseUpdates {
 					stmt.execute("CREATE INDEX AccountBalancesAssetBalanceIndex ON AccountBalances (asset_id, balance)");
 					// Add CHECK constraint to account balances
 					stmt.execute("ALTER TABLE AccountBalances ADD CONSTRAINT CheckBalanceNotNegative CHECK (balance >= 0)");
+					// Use a separate table space as this table will be very large.
+					stmt.execute("SET TABLE AccountBalances NEW SPACE");
 
 					// Keeping track of QORT gained from holding legacy QORA
 					stmt.execute("CREATE TABLE AccountQortFromQoraInfo (account QortalAddress, final_qort_from_qora QortalAmount, final_block_height INT, "
@@ -417,6 +421,8 @@ public class HSQLDBDatabaseUpdates {
 							+ "PRIMARY KEY (AT_address, height), FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)");
 					// For finding per-block AT states, ordered by creation timestamp
 					stmt.execute("CREATE INDEX BlockATStateIndex on ATStates (height, created_when)");
+					// Use a separate table space as this table will be very large.
+					stmt.execute("SET TABLE ATStates NEW SPACE");
 
 					// Deploy CIYAM AT Transactions
 					stmt.execute("CREATE TABLE DeployATTransactions (signature Signature, creator QortalPublicKey NOT NULL, AT_name ATName NOT NULL, "
@@ -653,6 +659,12 @@ public class HSQLDBDatabaseUpdates {
 					stmt.execute("DROP TABLE IF EXISTS NextBlockHeight");
 					break;
 
+				case 25:
+					// Remove excess created_when from ATStates
+					stmt.execute("ALTER TABLE ATStates DROP created_when");
+					stmt.execute("CREATE INDEX ATStateHeightIndex on ATStates (height)");
+					break;
+
 				default:
 					// nothing to do
 					return false;

src/main/java/org/qortal/settings/Settings.java

@@ -79,6 +79,8 @@ public class Settings {
 	private long repositoryBackupInterval = 0; // ms
 	/** Whether to show a notification when we backup repository. */
 	private boolean showBackupNotification = false;
+	/** How long to keep old, full, AT state data (ms). */
+	private long atStatesMaxLifetime = 2 * 7 * 24 * 60 * 60 * 1000L; // milliseconds
 
 	// Peer-to-peer related
 	private boolean isTestNet = false;
@@ -406,4 +408,8 @@ public class Settings {
 		return this.showBackupNotification;
 	}
 
+	public long getAtStatesMaxLifetime() {
+		return this.atStatesMaxLifetime;
+	}
+
 }