Merge branch 'at-states-fix'

# Conflicts:
#	src/main/java/org/qortal/controller/repository/AtStatesPruner.java
#	src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java

src/main/java/org/qortal/controller/repository/AtStatesPruner.java

@@ -39,9 +39,10 @@ public class AtStatesPruner implements Runnable {
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			int pruneStartHeight = repository.getATRepository().getAtPruneHeight();
+			int maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
 
 			repository.discardChanges();
-			repository.getATRepository().rebuildLatestAtStates();
+			repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
 			repository.saveChanges();
 
 			while (!Controller.isStopping()) {
@@ -92,7 +93,8 @@ public class AtStatesPruner implements Runnable {
 					if (upperPrunableHeight > upperBatchHeight) {
 						pruneStartHeight = upperBatchHeight;
 						repository.getATRepository().setAtPruneHeight(pruneStartHeight);
-						repository.getATRepository().rebuildLatestAtStates();
+						maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
+						repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
 						repository.saveChanges();
 
 						final int finalPruneStartHeight = pruneStartHeight;

src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java

@@ -26,9 +26,10 @@ public class AtStatesTrimmer implements Runnable {
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			int trimStartHeight = repository.getATRepository().getAtTrimHeight();
+			int maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
 
 			repository.discardChanges();
-			repository.getATRepository().rebuildLatestAtStates();
+			repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
 			repository.saveChanges();
 
 			while (!Controller.isStopping()) {
@@ -70,7 +71,8 @@ public class AtStatesTrimmer implements Runnable {
 					if (upperTrimmableHeight > upperBatchHeight) {
 						trimStartHeight = upperBatchHeight;
 						repository.getATRepository().setAtTrimHeight(trimStartHeight);
-						repository.getATRepository().rebuildLatestAtStates();
+						maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
+						repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
 						repository.saveChanges();
 
 						final int finalTrimStartHeight = trimStartHeight;

src/main/java/org/qortal/controller/repository/PruneManager.java

@@ -157,4 +157,18 @@ public class PruneManager {
         return (height < latestUnprunedHeight);
     }
 
+    /**
+     * When rebuilding the latest AT states, we need to specify a maxHeight, so that we aren't tracking
+     * very recent AT states that could potentially be orphaned. This method ensures that AT states
+     * are given a sufficient number of blocks to confirm before being tracked as a latest AT state.
+     */
+    public static int getMaxHeightForLatestAtStates(Repository repository) throws DataException {
+        // Get current chain height, and subtract a certain number of "confirmation" blocks
+        // This is to ensure we are basing our latest AT states data on confirmed blocks -
+        // ones that won't be orphaned in any normal circumstances
+        final int confirmationBlocks = 250;
+        final int chainHeight = repository.getBlockRepository().getBlockchainHeight();
+        return chainHeight - confirmationBlocks;
+    }
+
 }

src/main/java/org/qortal/repository/ATRepository.java

@@ -119,7 +119,7 @@ public interface ATRepository {
 	 * <p>
 	 * NOTE: performs implicit <tt>repository.saveChanges()</tt>.
 	 */
-	public void rebuildLatestAtStates() throws DataException;
+	public void rebuildLatestAtStates(int maxHeight) throws DataException;
 
 
 	/** Returns height of first trimmable AT state. */

src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java

@@ -603,7 +603,7 @@ public class HSQLDBATRepository implements ATRepository {
 
 
 	@Override
-	public void rebuildLatestAtStates() throws DataException {
+	public void rebuildLatestAtStates(int maxHeight) throws DataException {
 		// latestATStatesLock is to prevent concurrent updates on LatestATStates
 		// that could result in one process using a partial or empty dataset
 		// because it was in the process of being rebuilt by another thread
@@ -624,11 +624,12 @@ public class HSQLDBATRepository implements ATRepository {
 					+ "CROSS JOIN LATERAL("
 					+ "SELECT height FROM ATStates "
 					+ "WHERE ATStates.AT_address = ATs.AT_address "
+					+ "AND height <= ?"
 					+ "ORDER BY AT_address DESC, height DESC LIMIT 1"
 					+ ") "
 					+ ")";
 			try {
-				this.repository.executeCheckedUpdate(insertSql);
+				this.repository.executeCheckedUpdate(insertSql, maxHeight);
 			} catch (SQLException e) {
 				repository.examineException(e);
 				throw new DataException("Unable to populate temporary latest AT states cache in repository", e);

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java

@@ -99,7 +99,7 @@ public class HSQLDBDatabasePruning {
 
         // It's essential that we rebuild the latest AT states here, as we are using this data in the next query.
         // Failing to do this will result in important AT states being deleted, rendering the database unusable.
-        repository.getATRepository().rebuildLatestAtStates();
+        repository.getATRepository().rebuildLatestAtStates(endHeight);
 
 
         // Loop through all the LatestATStates and copy them to the new table

src/test/java/org/qortal/test/BlockArchiveTests.java

@@ -23,7 +23,6 @@ import org.qortal.transform.TransformationException;
 import org.qortal.transform.block.BlockTransformation;
 import org.qortal.utils.BlockArchiveUtils;
 import org.qortal.utils.NTP;
-import org.qortal.utils.Triple;
 
 import java.io.File;
 import java.io.IOException;
@@ -314,9 +313,10 @@ public class BlockArchiveTests extends Common {
 			repository.getBlockRepository().setBlockPruneHeight(901);
 
 			// Prune the AT states for the archived blocks
-			repository.getATRepository().rebuildLatestAtStates();
+			repository.getATRepository().rebuildLatestAtStates(900);
+			repository.saveChanges();
 			int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 900);
-			assertEquals(900-1, numATStatesPruned);
+			assertEquals(900-2, numATStatesPruned); // Minus 1 for genesis block, and another for the latest AT state
 			repository.getATRepository().setAtPruneHeight(901);
 
 			// Now ensure the SQL repository is missing blocks 2 and 900...
@@ -563,16 +563,23 @@ public class BlockArchiveTests extends Common {
 			// Trim the first 500 blocks
 			repository.getBlockRepository().trimOldOnlineAccountsSignatures(0, 500);
 			repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(501);
+			repository.getATRepository().rebuildLatestAtStates(500);
 			repository.getATRepository().trimAtStates(0, 500, 1000);
 			repository.getATRepository().setAtTrimHeight(501);
 
-			// Now block 500 should only have the AT state data hash
+			// Now block 499 should only have the AT state data hash
-			block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500);
+			List<ATStateData> block499AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(499);
-			atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500);
+			atStatesData = repository.getATRepository().getATStateAtHeight(block499AtStatesData.get(0).getATAddress(), 499);
 			assertNotNull(atStatesData.getStateHash());
 			assertNull(atStatesData.getStateData());
 
-			// ... but block 501 should have the full data
+			// ... but block 500 should have the full data (due to being retained as the "latest" AT state in the trimmed range
+			block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500);
+			atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500);
+			assertNotNull(atStatesData.getStateHash());
+			assertNotNull(atStatesData.getStateData());
+
+			// ... and block 501 should also have the full data
 			List<ATStateData> block501AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(501);
 			atStatesData = repository.getATRepository().getATStateAtHeight(block501AtStatesData.get(0).getATAddress(), 501);
 			assertNotNull(atStatesData.getStateHash());
@@ -612,9 +619,10 @@ public class BlockArchiveTests extends Common {
 			repository.getBlockRepository().setBlockPruneHeight(501);
 
 			// Prune the AT states for the archived blocks
-			repository.getATRepository().rebuildLatestAtStates();
+			repository.getATRepository().rebuildLatestAtStates(500);
+			repository.saveChanges();
 			int numATStatesPruned = repository.getATRepository().pruneAtStates(2, 500);
-			assertEquals(499, numATStatesPruned);
+			assertEquals(498, numATStatesPruned); // Minus 1 for genesis block, and another for the latest AT state
 			repository.getATRepository().setAtPruneHeight(501);
 
 			// Now ensure the SQL repository is missing blocks 2 and 500...

src/test/java/org/qortal/test/BootstrapTests.java

@@ -176,7 +176,8 @@ public class BootstrapTests extends Common {
         repository.getBlockRepository().setBlockPruneHeight(901);
 
         // Prune the AT states for the archived blocks
-        repository.getATRepository().rebuildLatestAtStates();
+        repository.getATRepository().rebuildLatestAtStates(900);
+        repository.saveChanges();
         repository.getATRepository().pruneAtStates(0, 900);
         repository.getATRepository().setAtPruneHeight(901);
 

src/test/java/org/qortal/test/PruneTests.java

@@ -1,16 +1,33 @@
 package org.qortal.test;
 
+import com.google.common.hash.HashCode;
 import org.junit.Before;
 import org.junit.Test;
+import org.qortal.account.Account;
 import org.qortal.account.PrivateKeyAccount;
+import org.qortal.asset.Asset;
+import org.qortal.block.Block;
 import org.qortal.controller.BlockMinter;
+import org.qortal.crosschain.AcctMode;
+import org.qortal.crosschain.LitecoinACCTv3;
+import org.qortal.data.at.ATData;
 import org.qortal.data.at.ATStateData;
 import org.qortal.data.block.BlockData;
+import org.qortal.data.crosschain.CrossChainTradeData;
+import org.qortal.data.transaction.BaseTransactionData;
+import org.qortal.data.transaction.DeployAtTransactionData;
+import org.qortal.data.transaction.MessageTransactionData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.group.Group;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
 import org.qortal.test.common.AtUtils;
+import org.qortal.test.common.BlockUtils;
 import org.qortal.test.common.Common;
+import org.qortal.test.common.TransactionUtils;
+import org.qortal.transaction.DeployAtTransaction;
+import org.qortal.transaction.MessageTransaction;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -19,6 +36,13 @@ import static org.junit.Assert.*;
 
 public class PruneTests extends Common {
 
+    // Constants for test AT (an LTC ACCT)
+    public static final byte[] litecoinPublicKeyHash = HashCode.fromString("bb00bb11bb22bb33bb44bb55bb66bb77bb88bb99").asBytes();
+    public static final int tradeTimeout = 20; // blocks
+    public static final long redeemAmount = 80_40200000L;
+    public static final long fundingAmount = 123_45600000L;
+    public static final long litecoinAmount = 864200L; // 0.00864200 LTC
+
     @Before
     public void beforeTest() throws DataException {
         Common.useDefaultSettings();
@@ -62,23 +86,32 @@ public class PruneTests extends Common {
             repository.getBlockRepository().setBlockPruneHeight(6);
 
             // Prune AT states for blocks 2-5
+            repository.getATRepository().rebuildLatestAtStates(5);
+            repository.saveChanges();
             int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 5);
-            assertEquals(4, numATStatesPruned);
+            assertEquals(3, numATStatesPruned);
             repository.getATRepository().setAtPruneHeight(6);
 
-            // Make sure that blocks 2-5 are now missing block data and AT states data
+            // Make sure that blocks 2-4 are now missing block data and AT states data
-            for (Integer i=2; i <= 5; i++) {
+            for (Integer i=2; i <= 4; i++) {
                 BlockData blockData = repository.getBlockRepository().fromHeight(i);
                 assertNull(blockData);
                 List<ATStateData> atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i);
                 assertTrue(atStatesDataList.isEmpty());
             }
 
-            // ... but blocks 6-10 have block data and full AT states data
+            // Block 5 should have full AT states data even though it was pruned.
+            // This is because we identified that as the "latest" AT state in that block range
+            BlockData blockData = repository.getBlockRepository().fromHeight(5);
+            assertNull(blockData);
+            List<ATStateData> atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(5);
+            assertEquals(1, atStatesDataList.size());
+
+            // Blocks 6-10 have block data and full AT states data
             for (Integer i=6; i <= 10; i++) {
-                BlockData blockData = repository.getBlockRepository().fromHeight(i);
+                blockData = repository.getBlockRepository().fromHeight(i);
                 assertNotNull(blockData.getSignature());
-                List<ATStateData> atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i);
+                atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i);
                 assertNotNull(atStatesDataList);
                 assertFalse(atStatesDataList.isEmpty());
                 ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(atStatesDataList.get(0).getATAddress(), i);
@@ -88,4 +121,102 @@ public class PruneTests extends Common {
         }
     }
 
+    @Test
+    public void testPruneSleepingAt() throws DataException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe");
+            PrivateKeyAccount tradeAccount = Common.getTestAccount(repository, "alice");
+
+            DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress());
+            Account at = deployAtTransaction.getATAccount();
+            String atAddress = at.getAddress();
+
+            // Mint enough blocks to take the original DEPLOY_AT past the prune threshold (in this case 20)
+            Block block = BlockUtils.mintBlocks(repository, 25);
+
+            // Send creator's address to AT, instead of typical partner's address
+            byte[] messageData = LitecoinACCTv3.getInstance().buildCancelMessage(deployer.getAddress());
+            long txTimestamp = block.getBlockData().getTimestamp();
+            MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress, txTimestamp);
+
+            // AT should process 'cancel' message in next block
+            BlockUtils.mintBlock(repository);
+
+            // Prune AT states up to block 20
+            repository.getATRepository().rebuildLatestAtStates(20);
+            repository.saveChanges();
+            int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 20);
+            assertEquals(1, numATStatesPruned); // deleted state at heights 2, but state at height 3 remains
+
+            // Check AT is finished
+            ATData atData = repository.getATRepository().fromATAddress(atAddress);
+            assertTrue(atData.getIsFinished());
+
+            // AT should be in CANCELLED mode
+            CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData);
+            assertEquals(AcctMode.CANCELLED, tradeData.mode);
+
+            // Test orphaning - should be possible because the previous AT state at height 3 is still available
+            BlockUtils.orphanLastBlock(repository);
+        }
+    }
+
+
+    // Helper methods for AT testing
+    private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, String tradeAddress) throws DataException {
+        byte[] creationBytes = LitecoinACCTv3.buildQortalAT(tradeAddress, litecoinPublicKeyHash, redeemAmount, litecoinAmount, tradeTimeout);
+
+        long txTimestamp = System.currentTimeMillis();
+        byte[] lastReference = deployer.getLastReference();
+
+        if (lastReference == null) {
+            System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress()));
+            System.exit(2);
+        }
+
+        Long fee = null;
+        String name = "QORT-LTC cross-chain trade";
+        String description = String.format("Qortal-Litecoin cross-chain trade");
+        String atType = "ACCT";
+        String tags = "QORT-LTC ACCT";
+
+        BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null);
+        TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT);
+
+        DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData);
+
+        fee = deployAtTransaction.calcRecommendedFee();
+        deployAtTransactionData.setFee(fee);
+
+        TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer);
+
+        return deployAtTransaction;
+    }
+
+    private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient, long txTimestamp) throws DataException {
+        byte[] lastReference = sender.getLastReference();
+
+        if (lastReference == null) {
+            System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress()));
+            System.exit(2);
+        }
+
+        Long fee = null;
+        int version = 4;
+        int nonce = 0;
+        long amount = 0;
+        Long assetId = null; // because amount is zero
+
+        BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null);
+        TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false);
+
+        MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData);
+
+        fee = messageTransaction.calcRecommendedFee();
+        messageTransactionData.setFee(fee);
+
+        TransactionUtils.signAndMint(repository, messageTransactionData, sender);
+
+        return messageTransaction;
+    }
 }

src/test/java/org/qortal/test/at/AtRepositoryTests.java

@@ -2,29 +2,20 @@ package org.qortal.test.at;
 
 import static org.junit.Assert.*;
 
-import java.nio.ByteBuffer;
 import java.util.List;
 
-import org.ciyam.at.CompilationException;
 import org.ciyam.at.MachineState;
-import org.ciyam.at.OpCode;
 import org.junit.Before;
 import org.junit.Test;
 import org.qortal.account.PrivateKeyAccount;
-import org.qortal.asset.Asset;
 import org.qortal.data.at.ATData;
 import org.qortal.data.at.ATStateData;
-import org.qortal.data.transaction.BaseTransactionData;
-import org.qortal.data.transaction.DeployAtTransactionData;
-import org.qortal.data.transaction.TransactionData;
-import org.qortal.group.Group;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
 import org.qortal.test.common.AtUtils;
 import org.qortal.test.common.BlockUtils;
 import org.qortal.test.common.Common;
-import org.qortal.test.common.TransactionUtils;
 import org.qortal.transaction.DeployAtTransaction;
 
 public class AtRepositoryTests extends Common {
@@ -76,7 +67,7 @@ public class AtRepositoryTests extends Common {
 			Integer testHeight = maxHeight - 2;
 
 			// Trim AT state data
-			repository.getATRepository().rebuildLatestAtStates();
+			repository.getATRepository().rebuildLatestAtStates(maxHeight);
 			repository.getATRepository().trimAtStates(2, maxHeight, 1000);
 
 			ATStateData atStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight);
@@ -130,7 +121,7 @@ public class AtRepositoryTests extends Common {
 			Integer testHeight = blockchainHeight;
 
 			// Trim AT state data
-			repository.getATRepository().rebuildLatestAtStates();
+			repository.getATRepository().rebuildLatestAtStates(maxHeight);
 			// COMMIT to check latest AT states persist / TEMPORARY table interaction
 			repository.saveChanges();
 
@@ -163,8 +154,8 @@ public class AtRepositoryTests extends Common {
 			int maxTrimHeight = blockchainHeight - 4;
 			Integer testHeight = maxTrimHeight + 1;
 
-			// Trim AT state data
+			// Trim AT state data (using a max height of maxTrimHeight + 1, so it is beyond the trimmed range)
-			repository.getATRepository().rebuildLatestAtStates();
+			repository.getATRepository().rebuildLatestAtStates(maxTrimHeight + 1);
 			repository.saveChanges();
 			repository.getATRepository().trimAtStates(2, maxTrimHeight, 1000);
 
@@ -333,7 +324,7 @@ public class AtRepositoryTests extends Common {
 			Integer testHeight = maxHeight - 2;
 
 			// Trim AT state data
-			repository.getATRepository().rebuildLatestAtStates();
+			repository.getATRepository().rebuildLatestAtStates(maxHeight);
 			repository.getATRepository().trimAtStates(2, maxHeight, 1000);
 
 			List<ATStateData> atStates = repository.getATRepository().getBlockATStatesAtHeight(testHeight);

src/test/java/org/qortal/test/common/BlockUtils.java

@@ -20,6 +20,15 @@ public class BlockUtils {
 		return BlockMinter.mintTestingBlock(repository, mintingAccount);
 	}
 
+	/** Mints multiple blocks using "alice-reward-share" test account, and returns the final block. */
+	public static Block mintBlocks(Repository repository, int count) throws DataException {
+		Block block = null;
+		for (int i=0; i<count; i++) {
+			block = BlockUtils.mintBlock(repository);
+		}
+		return block;
+	}
+
 	public static Long getNextBlockReward(Repository repository) throws DataException {
 		int currentHeight = repository.getBlockRepository().getBlockchainHeight();