From 896d8143854a43bb22d52f9b72c19e88b90154ec Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 12 Feb 2023 13:20:23 +0000 Subject: [PATCH] Add block_sequence to Transactions table, and populate all past transactions. This data was being lost when pruning the BlockTransactions table. Note: on first run this will reshape the db, which can take several minutes. --- src/main/java/org/qortal/block/Block.java | 11 ++- .../org/qortal/controller/Controller.java | 1 + .../data/transaction/TransactionData.java | 14 ++++ .../qortal/repository/RepositoryManager.java | 68 +++++++++++++++++++ .../repository/TransactionRepository.java | 2 + .../hsqldb/HSQLDBDatabaseUpdates.java | 11 +++ .../HSQLDBTransactionRepository.java | 20 +++++- 7 files changed, 123 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 3f306b93..59de8870 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1682,12 +1682,14 @@ public class Block { transactionData.getSignature()); this.repository.getBlockRepository().save(blockTransactionData); - // Update transaction's height in repository + // Update transaction's height in repository and local transactionData transactionRepository.updateBlockHeight(transactionData.getSignature(), this.blockData.getHeight()); - - // Update local transactionData's height too transaction.getTransactionData().setBlockHeight(this.blockData.getHeight()); + // Update transaction's sequence in repository and local transactionData + transactionRepository.updateBlockSequence(transactionData.getSignature(), sequence); + transaction.getTransactionData().setBlockSequence(sequence); + // No longer unconfirmed transactionRepository.confirmTransaction(transactionData.getSignature()); @@ -1774,6 +1776,9 @@ public class Block { // Unset height transactionRepository.updateBlockHeight(transactionData.getSignature(), null); + + // Unset sequence + transactionRepository.updateBlockSequence(transactionData.getSignature(), null); } transactionRepository.deleteParticipants(transactionData); diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index e9e1fcc2..054e5530 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -404,6 +404,7 @@ public class Controller extends Thread { try (final Repository repository = RepositoryManager.getRepository()) { RepositoryManager.archive(repository); RepositoryManager.prune(repository); + RepositoryManager.rebuildTransactionSequences(repository); } } catch (DataException e) { // If exception has no cause then repository is in use by some other process. diff --git a/src/main/java/org/qortal/data/transaction/TransactionData.java b/src/main/java/org/qortal/data/transaction/TransactionData.java index ec1139f4..713f98b5 100644 --- a/src/main/java/org/qortal/data/transaction/TransactionData.java +++ b/src/main/java/org/qortal/data/transaction/TransactionData.java @@ -76,6 +76,10 @@ public abstract class TransactionData { @Schema(accessMode = AccessMode.READ_ONLY, hidden = true, description = "height of block containing transaction") protected Integer blockHeight; + // Not always present + @Schema(accessMode = AccessMode.READ_ONLY, hidden = true, description = "sequence in block containing transaction") + protected Integer blockSequence; + // Not always present @Schema(accessMode = AccessMode.READ_ONLY, description = "group-approval status") protected ApprovalStatus approvalStatus; @@ -106,6 +110,7 @@ public abstract class TransactionData { this.fee = baseTransactionData.fee; this.signature = baseTransactionData.signature; this.blockHeight = baseTransactionData.blockHeight; + this.blockSequence = baseTransactionData.blockSequence; this.approvalStatus = baseTransactionData.approvalStatus; this.approvalHeight = baseTransactionData.approvalHeight; } @@ -174,6 +179,15 @@ public abstract class TransactionData { this.blockHeight = blockHeight; } + public Integer getBlockSequence() { + return this.blockSequence; + } + + @XmlTransient + public void setBlockSequence(Integer blockSequence) { + this.blockSequence = blockSequence; + } + public ApprovalStatus getApprovalStatus() { return approvalStatus; } diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index 0d9325b9..983404c1 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -2,13 +2,18 @@ package org.qortal.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.data.transaction.TransactionData; import org.qortal.gui.SplashFrame; import org.qortal.repository.hsqldb.HSQLDBDatabaseArchiving; import org.qortal.repository.hsqldb.HSQLDBDatabasePruning; import org.qortal.repository.hsqldb.HSQLDBRepository; import org.qortal.settings.Settings; +import org.qortal.transaction.Transaction; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.TimeoutException; public abstract class RepositoryManager { @@ -117,6 +122,69 @@ public abstract class RepositoryManager { return false; } + public static boolean rebuildTransactionSequences(Repository repository) throws DataException { + if (Settings.getInstance().isLite()) { + // Lite nodes have no blockchain + return false; + } + + try { + // Check if we have any unpopulated block_sequence values for the first 1000 blocks + List testSignatures = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria( + null, Arrays.asList("block_height < 1000 AND block_sequence IS NULL"), new ArrayList<>()); + if (testSignatures.isEmpty()) { + // block_sequence already populated for the first 1000 blocks, so assume complete. + // We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so + // we shouldn't ever be left in a partially rebuilt state. + return false; + } + + int blockchainHeight = repository.getBlockRepository().getBlockchainHeight(); + int totalTransactionCount = 0; + + for (int height = 1; height < blockchainHeight; height++) { + List transactions = new ArrayList<>(); + + // Fetch transactions for height + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, height, height); + for (byte[] signature : signatures) { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (transactionData != null) { + transactions.add(transactionData); + } + } + totalTransactionCount += transactions.size(); + + // Sort the transactions for this height + transactions.sort(Transaction.getDataComparator()); + + // Loop through and update sequences + for (int sequence = 0; sequence < transactions.size(); ++sequence) { + TransactionData transactionData = transactions.get(sequence); + + // Update transaction's sequence in repository + repository.getTransactionRepository().updateBlockSequence(transactionData.getSignature(), sequence); + } + + if (height % 10000 == 0) { + LOGGER.info("Rebuilt sequences for {} blocks (total transactions: {})", height, totalTransactionCount); + } + + repository.saveChanges(); + } + + LOGGER.info("Completed rebuild of transaction sequences."); + return true; + } + catch (DataException e) { + LOGGER.info("Unable to rebuild transaction sequences: {}. The database may have been left in an inconsistent state.", e.getMessage()); + + // Throw an exception so that the node startup is halted, allowing for a retry next time. + repository.discardChanges(); + throw new DataException("Rebuild of transaction sequences failed."); + } + } + public static void setRequestedCheckpoint(Boolean quick) { quickCheckpointRequested = quick; } diff --git a/src/main/java/org/qortal/repository/TransactionRepository.java b/src/main/java/org/qortal/repository/TransactionRepository.java index 105a317d..e528166b 100644 --- a/src/main/java/org/qortal/repository/TransactionRepository.java +++ b/src/main/java/org/qortal/repository/TransactionRepository.java @@ -309,6 +309,8 @@ public interface TransactionRepository { public void updateBlockHeight(byte[] signature, Integer height) throws DataException; + public void updateBlockSequence(byte[] signature, Integer sequence) throws DataException; + public void updateApprovalHeight(byte[] signature, Integer approvalHeight) throws DataException; /** diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index aecac034..cd2b30fa 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -993,6 +993,17 @@ public class HSQLDBDatabaseUpdates { stmt.execute("ALTER TABLE CancelSellNameTransactions ADD sale_price QortalAmount"); break; + case 47: + // Add `block_sequence` to the Transaction table, as the BlockTransactions table is pruned for + // older blocks and therefore the sequence becomes unavailable + LOGGER.info("Reshaping Transactions table - this can take a while..."); + stmt.execute("ALTER TABLE Transactions ADD block_sequence INTEGER"); + + // For finding transactions by height and sequence + LOGGER.info("Adding index to Transactions table - this can take a while..."); + stmt.execute("CREATE INDEX TransactionHeightSequenceIndex on Transactions (block_height, block_sequence)"); + break; + default: // nothing to do return false; diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index a8df1ab5..5bf149b2 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -657,8 +657,13 @@ public class HSQLDBTransactionRepository implements TransactionRepository { List bindParams) throws DataException { List signatures = new ArrayList<>(); + String txTypeClassName = ""; + if (txType != null) { + txTypeClassName = txType.className; + } + StringBuilder sql = new StringBuilder(1024); - sql.append(String.format("SELECT signature FROM %sTransactions", txType.className)); + sql.append(String.format("SELECT signature FROM %sTransactions", txTypeClassName)); if (!whereClauses.isEmpty()) { sql.append(" WHERE "); @@ -1444,6 +1449,19 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } } + @Override + public void updateBlockSequence(byte[] signature, Integer blockSequence) throws DataException { + HSQLDBSaver saver = new HSQLDBSaver("Transactions"); + + saver.bind("signature", signature).bind("block_sequence", blockSequence); + + try { + saver.execute(repository); + } catch (SQLException e) { + throw new DataException("Unable to update transaction's block sequence in repository", e); + } + } + @Override public void updateApprovalHeight(byte[] signature, Integer approvalHeight) throws DataException { HSQLDBSaver saver = new HSQLDBSaver("Transactions");