AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Rework of AT state trimming and pruning, in order to more reliably track the "latest" AT states.

Browse Source 
This should fix an edge case where AT states data was pruned/trimmed but it was then later required in consensus. The older state was deleted because it was replaced by a new "latest" state in a brand new block. But once the new "latest" state was orphaned from the block, the old "latest" state was then required again.

This works around the problem by excluding very recent blocks in the latest AT states data, so that it is unaffected by real-time sync activity.

The trade off is that we could end up retaining more AT states than needed, so a secondary cleanup process may need to run at some time in the future to remove these. But it should only be a minimal amount of data, and can be cleaned up with a single query. This would have been happening to a certain degree already.

# Conflicts:
#	src/main/java/org/qortal/controller/repository/AtStatesPruner.java
#	src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java
This commit is contained in:
CalDescent 2023-01-15 14:32:33 +00:00
parent 257ca2da05
commit ae5b713e58
11 changed files with 197 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/org/qortal/controller/repository/AtStatesPruner.java
View File

@ -39,9 +39,10 @@ public class AtStatesPruner implements Runnable {
try (final Repository repository = RepositoryManager.getRepository()) {
int pruneStartHeight = repository.getATRepository().getAtPruneHeight();
int maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
repository.discardChanges();
repository.getATRepository().rebuildLatestAtStates();
repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
repository.saveChanges();
while (!Controller.isStopping()) {
@ -92,7 +93,8 @@ public class AtStatesPruner implements Runnable {
if (upperPrunableHeight > upperBatchHeight) {
pruneStartHeight = upperBatchHeight;
repository.getATRepository().setAtPruneHeight(pruneStartHeight);
repository.getATRepository().rebuildLatestAtStates();
maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
repository.saveChanges();
final int finalPruneStartHeight = pruneStartHeight;

6
src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java
View File

@ -26,9 +26,10 @@ public class AtStatesTrimmer implements Runnable {
try (final Repository repository = RepositoryManager.getRepository()) {
int trimStartHeight = repository.getATRepository().getAtTrimHeight();
int maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
repository.discardChanges();
repository.getATRepository().rebuildLatestAtStates();
repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
repository.saveChanges();
while (!Controller.isStopping()) {
@ -70,7 +71,8 @@ public class AtStatesTrimmer implements Runnable {
if (upperTrimmableHeight > upperBatchHeight) {
trimStartHeight = upperBatchHeight;
repository.getATRepository().setAtTrimHeight(trimStartHeight);
repository.getATRepository().rebuildLatestAtStates();
maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
repository.saveChanges();
final int finalTrimStartHeight = trimStartHeight;

14
src/main/java/org/qortal/controller/repository/PruneManager.java
View File

@ -157,4 +157,18 @@ public class PruneManager {
return (height < latestUnprunedHeight);
}
/**
* When rebuilding the latest AT states, we need to specify a maxHeight, so that we aren't tracking
* very recent AT states that could potentially be orphaned. This method ensures that AT states
* are given a sufficient number of blocks to confirm before being tracked as a latest AT state.
*/
public static int getMaxHeightForLatestAtStates(Repository repository) throws DataException {
// Get current chain height, and subtract a certain number of "confirmation" blocks
// This is to ensure we are basing our latest AT states data on confirmed blocks -
// ones that won't be orphaned in any normal circumstances
final int confirmationBlocks = 250;
final int chainHeight = repository.getBlockRepository().getBlockchainHeight();
return chainHeight - confirmationBlocks;
}
}

2
src/main/java/org/qortal/repository/ATRepository.java
View File

@ -119,7 +119,7 @@ public interface ATRepository {
* <p>
* NOTE: performs implicit <tt>repository.saveChanges()</tt>.
*/
public void rebuildLatestAtStates() throws DataException;
public void rebuildLatestAtStates(int maxHeight) throws DataException;
/** Returns height of first trimmable AT state. */

5
src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java
View File

@ -603,7 +603,7 @@ public class HSQLDBATRepository implements ATRepository {
@Override
public void rebuildLatestAtStates() throws DataException {
public void rebuildLatestAtStates(int maxHeight) throws DataException {
// latestATStatesLock is to prevent concurrent updates on LatestATStates
// that could result in one process using a partial or empty dataset
// because it was in the process of being rebuilt by another thread
@ -624,11 +624,12 @@ public class HSQLDBATRepository implements ATRepository {
+ "CROSS JOIN LATERAL("
+ "SELECT height FROM ATStates "
+ "WHERE ATStates.AT_address = ATs.AT_address "
+ "AND height <= ?"
+ "ORDER BY AT_address DESC, height DESC LIMIT 1"
+ ") "
+ ")";
try {
this.repository.executeCheckedUpdate(insertSql);
this.repository.executeCheckedUpdate(insertSql, maxHeight);
} catch (SQLException e) {
repository.examineException(e);
throw new DataException("Unable to populate temporary latest AT states cache in repository", e);

2
src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java
View File

@ -99,7 +99,7 @@ public class HSQLDBDatabasePruning {
// It's essential that we rebuild the latest AT states here, as we are using this data in the next query.
// Failing to do this will result in important AT states being deleted, rendering the database unusable.
repository.getATRepository().rebuildLatestAtStates();
repository.getATRepository().rebuildLatestAtStates(endHeight);
// Loop through all the LatestATStates and copy them to the new table

26
src/test/java/org/qortal/test/BlockArchiveTests.java
View File

@ -23,7 +23,6 @@ import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.utils.BlockArchiveUtils;
import org.qortal.utils.NTP;
import org.qortal.utils.Triple;
import java.io.File;
import java.io.IOException;
@ -314,9 +313,10 @@ public class BlockArchiveTests extends Common {
repository.getBlockRepository().setBlockPruneHeight(901);
// Prune the AT states for the archived blocks
repository.getATRepository().rebuildLatestAtStates();
repository.getATRepository().rebuildLatestAtStates(900);
repository.saveChanges();
int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 900);
assertEquals(900-1, numATStatesPruned);
assertEquals(900-2, numATStatesPruned); // Minus 1 for genesis block, and another for the latest AT state
repository.getATRepository().setAtPruneHeight(901);
// Now ensure the SQL repository is missing blocks 2 and 900...
@ -563,16 +563,23 @@ public class BlockArchiveTests extends Common {
// Trim the first 500 blocks
repository.getBlockRepository().trimOldOnlineAccountsSignatures(0, 500);
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(501);
repository.getATRepository().rebuildLatestAtStates(500);
repository.getATRepository().trimAtStates(0, 500, 1000);
repository.getATRepository().setAtTrimHeight(501);
// Now block 500 should only have the AT state data hash
block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500);
atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500);
// Now block 499 should only have the AT state data hash
List<ATStateData> block499AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(499);
atStatesData = repository.getATRepository().getATStateAtHeight(block499AtStatesData.get(0).getATAddress(), 499);
assertNotNull(atStatesData.getStateHash());
assertNull(atStatesData.getStateData());
// ... but block 501 should have the full data
// ... but block 500 should have the full data (due to being retained as the "latest" AT state in the trimmed range
block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500);
atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500);
assertNotNull(atStatesData.getStateHash());
assertNotNull(atStatesData.getStateData());
// ... and block 501 should also have the full data
List<ATStateData> block501AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(501);
atStatesData = repository.getATRepository().getATStateAtHeight(block501AtStatesData.get(0).getATAddress(), 501);
assertNotNull(atStatesData.getStateHash());
@ -612,9 +619,10 @@ public class BlockArchiveTests extends Common {
repository.getBlockRepository().setBlockPruneHeight(501);
// Prune the AT states for the archived blocks
repository.getATRepository().rebuildLatestAtStates();
repository.getATRepository().rebuildLatestAtStates(500);
repository.saveChanges();
int numATStatesPruned = repository.getATRepository().pruneAtStates(2, 500);
assertEquals(499, numATStatesPruned);
assertEquals(498, numATStatesPruned); // Minus 1 for genesis block, and another for the latest AT state
repository.getATRepository().setAtPruneHeight(501);
// Now ensure the SQL repository is missing blocks 2 and 500...

3
src/test/java/org/qortal/test/BootstrapTests.java
View File

@ -176,7 +176,8 @@ public class BootstrapTests extends Common {
repository.getBlockRepository().setBlockPruneHeight(901);
// Prune the AT states for the archived blocks
repository.getATRepository().rebuildLatestAtStates();
repository.getATRepository().rebuildLatestAtStates(900);
repository.saveChanges();
repository.getATRepository().pruneAtStates(0, 900);
repository.getATRepository().setAtPruneHeight(901);

143
src/test/java/org/qortal/test/PruneTests.java
View File

@ -1,16 +1,33 @@
package org.qortal.test;
import com.google.common.hash.HashCode;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.Account;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.asset.Asset;
import org.qortal.block.Block;
import org.qortal.controller.BlockMinter;
import org.qortal.crosschain.AcctMode;
import org.qortal.crosschain.LitecoinACCTv3;
import org.qortal.data.at.ATData;
import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData;
import org.qortal.data.crosschain.CrossChainTradeData;
import org.qortal.data.transaction.BaseTransactionData;
import org.qortal.data.transaction.DeployAtTransactionData;
import org.qortal.data.transaction.MessageTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.group.Group;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.AtUtils;
import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.transaction.DeployAtTransaction;
import org.qortal.transaction.MessageTransaction;
import java.util.ArrayList;
import java.util.List;
@ -19,6 +36,13 @@ import static org.junit.Assert.*;
public class PruneTests extends Common {
// Constants for test AT (an LTC ACCT)
public static final byte[] litecoinPublicKeyHash = HashCode.fromString("bb00bb11bb22bb33bb44bb55bb66bb77bb88bb99").asBytes();
public static final int tradeTimeout = 20; // blocks
public static final long redeemAmount = 80_40200000L;
public static final long fundingAmount = 123_45600000L;
public static final long litecoinAmount = 864200L; // 0.00864200 LTC
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings();
@ -62,23 +86,32 @@ public class PruneTests extends Common {
repository.getBlockRepository().setBlockPruneHeight(6);
// Prune AT states for blocks 2-5
repository.getATRepository().rebuildLatestAtStates(5);
repository.saveChanges();
int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 5);
assertEquals(4, numATStatesPruned);
assertEquals(3, numATStatesPruned);
repository.getATRepository().setAtPruneHeight(6);
// Make sure that blocks 2-5 are now missing block data and AT states data
for (Integer i=2; i <= 5; i++) {
// Make sure that blocks 2-4 are now missing block data and AT states data
for (Integer i=2; i <= 4; i++) {
BlockData blockData = repository.getBlockRepository().fromHeight(i);
assertNull(blockData);
List<ATStateData> atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i);
assertTrue(atStatesDataList.isEmpty());
}
// ... but blocks 6-10 have block data and full AT states data
// Block 5 should have full AT states data even though it was pruned.
// This is because we identified that as the "latest" AT state in that block range
BlockData blockData = repository.getBlockRepository().fromHeight(5);
assertNull(blockData);
List<ATStateData> atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(5);
assertEquals(1, atStatesDataList.size());
// Blocks 6-10 have block data and full AT states data
for (Integer i=6; i <= 10; i++) {
BlockData blockData = repository.getBlockRepository().fromHeight(i);
blockData = repository.getBlockRepository().fromHeight(i);
assertNotNull(blockData.getSignature());
List<ATStateData> atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i);
atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i);
assertNotNull(atStatesDataList);
assertFalse(atStatesDataList.isEmpty());
ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(atStatesDataList.get(0).getATAddress(), i);
@ -88,4 +121,102 @@ public class PruneTests extends Common {
}
}
@Test
public void testPruneSleepingAt() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe");
PrivateKeyAccount tradeAccount = Common.getTestAccount(repository, "alice");
DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress());
Account at = deployAtTransaction.getATAccount();
String atAddress = at.getAddress();
// Mint enough blocks to take the original DEPLOY_AT past the prune threshold (in this case 20)
Block block = BlockUtils.mintBlocks(repository, 25);
// Send creator's address to AT, instead of typical partner's address
byte[] messageData = LitecoinACCTv3.getInstance().buildCancelMessage(deployer.getAddress());
long txTimestamp = block.getBlockData().getTimestamp();
MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress, txTimestamp);
// AT should process 'cancel' message in next block
BlockUtils.mintBlock(repository);
// Prune AT states up to block 20
repository.getATRepository().rebuildLatestAtStates(20);
repository.saveChanges();
int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 20);
assertEquals(1, numATStatesPruned); // deleted state at heights 2, but state at height 3 remains
// Check AT is finished
ATData atData = repository.getATRepository().fromATAddress(atAddress);
assertTrue(atData.getIsFinished());
// AT should be in CANCELLED mode
CrossChainTradeData tradeData = LitecoinACCTv3.getInstance().populateTradeData(repository, atData);
assertEquals(AcctMode.CANCELLED, tradeData.mode);
// Test orphaning - should be possible because the previous AT state at height 3 is still available
BlockUtils.orphanLastBlock(repository);
}
}
// Helper methods for AT testing
private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, String tradeAddress) throws DataException {
byte[] creationBytes = LitecoinACCTv3.buildQortalAT(tradeAddress, litecoinPublicKeyHash, redeemAmount, litecoinAmount, tradeTimeout);
long txTimestamp = System.currentTimeMillis();
byte[] lastReference = deployer.getLastReference();
if (lastReference == null) {
System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress()));
System.exit(2);
}
Long fee = null;
String name = "QORT-LTC cross-chain trade";
String description = String.format("Qortal-Litecoin cross-chain trade");
String atType = "ACCT";
String tags = "QORT-LTC ACCT";
BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null);
TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT);
DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData);
fee = deployAtTransaction.calcRecommendedFee();
deployAtTransactionData.setFee(fee);
TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer);
return deployAtTransaction;
}
private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient, long txTimestamp) throws DataException {
byte[] lastReference = sender.getLastReference();
if (lastReference == null) {
System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress()));
System.exit(2);
}
Long fee = null;
int version = 4;
int nonce = 0;
long amount = 0;
Long assetId = null; // because amount is zero
BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null);
TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false);
MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData);
fee = messageTransaction.calcRecommendedFee();
messageTransactionData.setFee(fee);
TransactionUtils.signAndMint(repository, messageTransactionData, sender);
return messageTransaction;
}
}

19
src/test/java/org/qortal/test/at/AtRepositoryTests.java
View File

@ -2,29 +2,20 @@ package org.qortal.test.at;
import static org.junit.Assert.*;
import java.nio.ByteBuffer;
import java.util.List;
import org.ciyam.at.CompilationException;
import org.ciyam.at.MachineState;
import org.ciyam.at.OpCode;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.asset.Asset;
import org.qortal.data.at.ATData;
import org.qortal.data.at.ATStateData;
import org.qortal.data.transaction.BaseTransactionData;
import org.qortal.data.transaction.DeployAtTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.group.Group;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.AtUtils;
import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.transaction.DeployAtTransaction;
public class AtRepositoryTests extends Common {
@ -76,7 +67,7 @@ public class AtRepositoryTests extends Common {
Integer testHeight = maxHeight - 2;
// Trim AT state data
repository.getATRepository().rebuildLatestAtStates();
repository.getATRepository().rebuildLatestAtStates(maxHeight);
repository.getATRepository().trimAtStates(2, maxHeight, 1000);
ATStateData atStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight);
@ -130,7 +121,7 @@ public class AtRepositoryTests extends Common {
Integer testHeight = blockchainHeight;
// Trim AT state data
repository.getATRepository().rebuildLatestAtStates();
repository.getATRepository().rebuildLatestAtStates(maxHeight);
// COMMIT to check latest AT states persist / TEMPORARY table interaction
repository.saveChanges();
@ -163,8 +154,8 @@ public class AtRepositoryTests extends Common {
int maxTrimHeight = blockchainHeight - 4;
Integer testHeight = maxTrimHeight + 1;
// Trim AT state data
repository.getATRepository().rebuildLatestAtStates();
// Trim AT state data (using a max height of maxTrimHeight + 1, so it is beyond the trimmed range)
repository.getATRepository().rebuildLatestAtStates(maxTrimHeight + 1);
repository.saveChanges();
repository.getATRepository().trimAtStates(2, maxTrimHeight, 1000);
@ -333,7 +324,7 @@ public class AtRepositoryTests extends Common {
Integer testHeight = maxHeight - 2;
// Trim AT state data
repository.getATRepository().rebuildLatestAtStates();
repository.getATRepository().rebuildLatestAtStates(maxHeight);
repository.getATRepository().trimAtStates(2, maxHeight, 1000);
List<ATStateData> atStates = repository.getATRepository().getBlockATStatesAtHeight(testHeight);

9
src/test/java/org/qortal/test/common/BlockUtils.java
View File

@ -20,6 +20,15 @@ public class BlockUtils {
return BlockMinter.mintTestingBlock(repository, mintingAccount);
}
/** Mints multiple blocks using "alice-reward-share" test account, and returns the final block. */
public static Block mintBlocks(Repository repository, int count) throws DataException {
Block block = null;
for (int i=0; i<count; i++) {
block = BlockUtils.mintBlock(repository);
}
return block;
}
public static Long getNextBlockReward(Repository repository) throws DataException {
int currentHeight = repository.getBlockRepository().getBlockchainHeight();