diff --git a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java index bd12f784..1faeda98 100644 --- a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java @@ -39,9 +39,10 @@ public class AtStatesPruner implements Runnable { try (final Repository repository = RepositoryManager.getRepository()) { int pruneStartHeight = repository.getATRepository().getAtPruneHeight(); + int maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository); repository.discardChanges(); - repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight); while (!Controller.isStopping()) { repository.discardChanges(); @@ -91,7 +92,8 @@ public class AtStatesPruner implements Runnable { if (upperPrunableHeight > upperBatchHeight) { pruneStartHeight = upperBatchHeight; repository.getATRepository().setAtPruneHeight(pruneStartHeight); - repository.getATRepository().rebuildLatestAtStates(); + maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository); + repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight); repository.saveChanges(); final int finalPruneStartHeight = pruneStartHeight; diff --git a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java index 69fa347c..ea56699c 100644 --- a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java @@ -26,9 +26,10 @@ public class AtStatesTrimmer implements Runnable { try (final Repository repository = RepositoryManager.getRepository()) { int trimStartHeight = repository.getATRepository().getAtTrimHeight(); + int maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository); repository.discardChanges(); - repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight); while (!Controller.isStopping()) { repository.discardChanges(); @@ -69,7 +70,8 @@ public class AtStatesTrimmer implements Runnable { if (upperTrimmableHeight > upperBatchHeight) { trimStartHeight = upperBatchHeight; repository.getATRepository().setAtTrimHeight(trimStartHeight); - repository.getATRepository().rebuildLatestAtStates(); + maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository); + repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight); repository.saveChanges(); final int finalTrimStartHeight = trimStartHeight; diff --git a/src/main/java/org/qortal/controller/repository/PruneManager.java b/src/main/java/org/qortal/controller/repository/PruneManager.java index ec27456f..dfb6290b 100644 --- a/src/main/java/org/qortal/controller/repository/PruneManager.java +++ b/src/main/java/org/qortal/controller/repository/PruneManager.java @@ -157,4 +157,18 @@ public class PruneManager { return (height < latestUnprunedHeight); } + /** + * When rebuilding the latest AT states, we need to specify a maxHeight, so that we aren't tracking + * very recent AT states that could potentially be orphaned. This method ensures that AT states + * are given a sufficient number of blocks to confirm before being tracked as a latest AT state. + */ + public static int getMaxHeightForLatestAtStates(Repository repository) throws DataException { + // Get current chain height, and subtract a certain number of "confirmation" blocks + // This is to ensure we are basing our latest AT states data on confirmed blocks - + // ones that won't be orphaned in any normal circumstances + final int confirmationBlocks = 250; + final int chainHeight = repository.getBlockRepository().getBlockchainHeight(); + return chainHeight - confirmationBlocks; + } + } diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 0f537ae9..93da924c 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -119,7 +119,7 @@ public interface ATRepository { *

* NOTE: performs implicit repository.saveChanges(). */ - public void rebuildLatestAtStates() throws DataException; + public void rebuildLatestAtStates(int maxHeight) throws DataException; /** Returns height of first trimmable AT state. */ diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index 04823925..dd0404a8 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -603,7 +603,7 @@ public class HSQLDBATRepository implements ATRepository { @Override - public void rebuildLatestAtStates() throws DataException { + public void rebuildLatestAtStates(int maxHeight) throws DataException { // latestATStatesLock is to prevent concurrent updates on LatestATStates // that could result in one process using a partial or empty dataset // because it was in the process of being rebuilt by another thread @@ -624,11 +624,12 @@ public class HSQLDBATRepository implements ATRepository { + "CROSS JOIN LATERAL(" + "SELECT height FROM ATStates " + "WHERE ATStates.AT_address = ATs.AT_address " + + "AND height <= ?" + "ORDER BY AT_address DESC, height DESC LIMIT 1" + ") " + ")"; try { - this.repository.executeCheckedUpdate(insertSql); + this.repository.executeCheckedUpdate(insertSql, maxHeight); } catch (SQLException e) { repository.examineException(e); throw new DataException("Unable to populate temporary latest AT states cache in repository", e); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java index 978ba25e..e2bfc9ef 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java @@ -99,7 +99,7 @@ public class HSQLDBDatabasePruning { // It's essential that we rebuild the latest AT states here, as we are using this data in the next query. // Failing to do this will result in important AT states being deleted, rendering the database unusable. - repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().rebuildLatestAtStates(endHeight); // Loop through all the LatestATStates and copy them to the new table diff --git a/src/test/java/org/qortal/test/BlockArchiveTests.java b/src/test/java/org/qortal/test/BlockArchiveTests.java index 3bfa4e84..8b3de67b 100644 --- a/src/test/java/org/qortal/test/BlockArchiveTests.java +++ b/src/test/java/org/qortal/test/BlockArchiveTests.java @@ -23,7 +23,6 @@ import org.qortal.transform.TransformationException; import org.qortal.transform.block.BlockTransformation; import org.qortal.utils.BlockArchiveUtils; import org.qortal.utils.NTP; -import org.qortal.utils.Triple; import java.io.File; import java.io.IOException; @@ -314,9 +313,10 @@ public class BlockArchiveTests extends Common { repository.getBlockRepository().setBlockPruneHeight(901); // Prune the AT states for the archived blocks - repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().rebuildLatestAtStates(900); + repository.saveChanges(); int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 900); - assertEquals(900-1, numATStatesPruned); + assertEquals(900-2, numATStatesPruned); // Minus 1 for genesis block, and another for the latest AT state repository.getATRepository().setAtPruneHeight(901); // Now ensure the SQL repository is missing blocks 2 and 900... @@ -563,16 +563,23 @@ public class BlockArchiveTests extends Common { // Trim the first 500 blocks repository.getBlockRepository().trimOldOnlineAccountsSignatures(0, 500); repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(501); + repository.getATRepository().rebuildLatestAtStates(500); repository.getATRepository().trimAtStates(0, 500, 1000); repository.getATRepository().setAtTrimHeight(501); - // Now block 500 should only have the AT state data hash - block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500); - atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500); + // Now block 499 should only have the AT state data hash + List block499AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(499); + atStatesData = repository.getATRepository().getATStateAtHeight(block499AtStatesData.get(0).getATAddress(), 499); assertNotNull(atStatesData.getStateHash()); assertNull(atStatesData.getStateData()); - // ... but block 501 should have the full data + // ... but block 500 should have the full data (due to being retained as the "latest" AT state in the trimmed range + block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500); + atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + + // ... and block 501 should also have the full data List block501AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(501); atStatesData = repository.getATRepository().getATStateAtHeight(block501AtStatesData.get(0).getATAddress(), 501); assertNotNull(atStatesData.getStateHash()); @@ -612,9 +619,10 @@ public class BlockArchiveTests extends Common { repository.getBlockRepository().setBlockPruneHeight(501); // Prune the AT states for the archived blocks - repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().rebuildLatestAtStates(500); + repository.saveChanges(); int numATStatesPruned = repository.getATRepository().pruneAtStates(2, 500); - assertEquals(499, numATStatesPruned); + assertEquals(498, numATStatesPruned); // Minus 1 for genesis block, and another for the latest AT state repository.getATRepository().setAtPruneHeight(501); // Now ensure the SQL repository is missing blocks 2 and 500... diff --git a/src/test/java/org/qortal/test/BootstrapTests.java b/src/test/java/org/qortal/test/BootstrapTests.java index aa641e71..b60b412c 100644 --- a/src/test/java/org/qortal/test/BootstrapTests.java +++ b/src/test/java/org/qortal/test/BootstrapTests.java @@ -176,7 +176,8 @@ public class BootstrapTests extends Common { repository.getBlockRepository().setBlockPruneHeight(901); // Prune the AT states for the archived blocks - repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().rebuildLatestAtStates(900); + repository.saveChanges(); repository.getATRepository().pruneAtStates(0, 900); repository.getATRepository().setAtPruneHeight(901); diff --git a/src/test/java/org/qortal/test/PruneTests.java b/src/test/java/org/qortal/test/PruneTests.java index 0914d794..5a31146e 100644 --- a/src/test/java/org/qortal/test/PruneTests.java +++ b/src/test/java/org/qortal/test/PruneTests.java @@ -1,16 +1,33 @@ package org.qortal.test; +import com.google.common.hash.HashCode; import org.junit.Before; import org.junit.Test; +import org.qortal.account.Account; import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.block.Block; import org.qortal.controller.BlockMinter; +import org.qortal.crosschain.AcctMode; +import org.qortal.crosschain.LitecoinACCTv3; +import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.data.block.BlockData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.test.common.AtUtils; +import org.qortal.test.common.BlockUtils; import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; import java.util.ArrayList; import java.util.List; @@ -19,6 +36,13 @@ import static org.junit.Assert.*; public class PruneTests extends Common { + // Constants for test AT (an LTC ACCT) + public static final byte[] litecoinPublicKeyHash = HashCode.fromString("bb00bb11bb22bb33bb44bb55bb66bb77bb88bb99").asBytes(); + public static final int tradeTimeout = 20; // blocks + public static final long redeemAmount = 80_40200000L; + public static final long fundingAmount = 123_45600000L; + public static final long litecoinAmount = 864200L; // 0.00864200 LTC + @Before public void beforeTest() throws DataException { Common.useDefaultSettings(); @@ -62,23 +86,32 @@ public class PruneTests extends Common { repository.getBlockRepository().setBlockPruneHeight(6); // Prune AT states for blocks 2-5 + repository.getATRepository().rebuildLatestAtStates(5); + repository.saveChanges(); int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 5); - assertEquals(4, numATStatesPruned); + assertEquals(3, numATStatesPruned); repository.getATRepository().setAtPruneHeight(6); - // Make sure that blocks 2-5 are now missing block data and AT states data - for (Integer i=2; i <= 5; i++) { + // Make sure that blocks 2-4 are now missing block data and AT states data + for (Integer i=2; i <= 4; i++) { BlockData blockData = repository.getBlockRepository().fromHeight(i); assertNull(blockData); List atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i); assertTrue(atStatesDataList.isEmpty()); } - // ... but blocks 6-10 have block data and full AT states data + // Block 5 should have full AT states data even though it was pruned. + // This is because we identified that as the "latest" AT state in that block range + BlockData blockData = repository.getBlockRepository().fromHeight(5); + assertNull(blockData); + List atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(5); + assertEquals(1, atStatesDataList.size()); + + // Blocks 6-10 have block data and full AT states data for (Integer i=6; i <= 10; i++) { - BlockData blockData = repository.getBlockRepository().fromHeight(i); + blockData = repository.getBlockRepository().fromHeight(i); assertNotNull(blockData.getSignature()); - List atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i); + atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i); assertNotNull(atStatesDataList); assertFalse(atStatesDataList.isEmpty()); ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(atStatesDataList.get(0).getATAddress(), i); @@ -88,4 +121,102 @@ public class PruneTests extends Common { } } + @Test + public void testPruneSleepingAt() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = Common.getTestAccount(repository, "alice"); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + // Mint enough blocks to take the original DEPLOY_AT past the prune threshold (in this case 20) + Block block = BlockUtils.mintBlocks(repository, 25); + + // Send creator's address to AT, instead of typical partner's address + byte[] messageData = LitecoinACCTv3.getInstance().buildCancelMessage(deployer.getAddress()); + long txTimestamp = block.getBlockData().getTimestamp(); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress, txTimestamp); + + // AT should process 'cancel' message in next block + BlockUtils.mintBlock(repository); + + // Prune AT states up to block 20 + repository.getATRepository().rebuildLatestAtStates(20); + repository.saveChanges(); + int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 20); + assertEquals(1, numATStatesPruned); // deleted state at heights 2, but state at height 3 remains + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + + // Test orphaning - should be possible because the previous AT state at height 3 is still available + BlockUtils.orphanLastBlock(repository); + } + } + + + // Helper methods for AT testing + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, String tradeAddress) throws DataException { + byte[] creationBytes = LitecoinACCTv3.buildQortalAT(tradeAddress, litecoinPublicKeyHash, redeemAmount, litecoinAmount, tradeTimeout); + + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "QORT-LTC cross-chain trade"; + String description = String.format("Qortal-Litecoin cross-chain trade"); + String atType = "ACCT"; + String tags = "QORT-LTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient, long txTimestamp) throws DataException { + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, messageTransactionData, sender); + + return messageTransaction; + } } diff --git a/src/test/java/org/qortal/test/at/AtRepositoryTests.java b/src/test/java/org/qortal/test/at/AtRepositoryTests.java index 8ef4c774..8441731f 100644 --- a/src/test/java/org/qortal/test/at/AtRepositoryTests.java +++ b/src/test/java/org/qortal/test/at/AtRepositoryTests.java @@ -2,29 +2,20 @@ package org.qortal.test.at; import static org.junit.Assert.*; -import java.nio.ByteBuffer; import java.util.List; -import org.ciyam.at.CompilationException; import org.ciyam.at.MachineState; -import org.ciyam.at.OpCode; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; -import org.qortal.asset.Asset; import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.group.Group; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.test.common.AtUtils; import org.qortal.test.common.BlockUtils; import org.qortal.test.common.Common; -import org.qortal.test.common.TransactionUtils; import org.qortal.transaction.DeployAtTransaction; public class AtRepositoryTests extends Common { @@ -76,7 +67,7 @@ public class AtRepositoryTests extends Common { Integer testHeight = maxHeight - 2; // Trim AT state data - repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().rebuildLatestAtStates(maxHeight); repository.getATRepository().trimAtStates(2, maxHeight, 1000); ATStateData atStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight); @@ -130,7 +121,7 @@ public class AtRepositoryTests extends Common { Integer testHeight = blockchainHeight; // Trim AT state data - repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().rebuildLatestAtStates(maxHeight); // COMMIT to check latest AT states persist / TEMPORARY table interaction repository.saveChanges(); @@ -163,8 +154,8 @@ public class AtRepositoryTests extends Common { int maxTrimHeight = blockchainHeight - 4; Integer testHeight = maxTrimHeight + 1; - // Trim AT state data - repository.getATRepository().rebuildLatestAtStates(); + // Trim AT state data (using a max height of maxTrimHeight + 1, so it is beyond the trimmed range) + repository.getATRepository().rebuildLatestAtStates(maxTrimHeight + 1); repository.saveChanges(); repository.getATRepository().trimAtStates(2, maxTrimHeight, 1000); @@ -333,7 +324,7 @@ public class AtRepositoryTests extends Common { Integer testHeight = maxHeight - 2; // Trim AT state data - repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().rebuildLatestAtStates(maxHeight); repository.getATRepository().trimAtStates(2, maxHeight, 1000); List atStates = repository.getATRepository().getBlockATStatesAtHeight(testHeight); diff --git a/src/test/java/org/qortal/test/common/BlockUtils.java b/src/test/java/org/qortal/test/common/BlockUtils.java index 3077b65b..ab57dadf 100644 --- a/src/test/java/org/qortal/test/common/BlockUtils.java +++ b/src/test/java/org/qortal/test/common/BlockUtils.java @@ -20,6 +20,15 @@ public class BlockUtils { return BlockMinter.mintTestingBlock(repository, mintingAccount); } + /** Mints multiple blocks using "alice-reward-share" test account, and returns the final block. */ + public static Block mintBlocks(Repository repository, int count) throws DataException { + Block block = null; + for (int i=0; i