diff --git a/pom.xml b/pom.xml index 0dfa0cf4..c9986fd4 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 4.0.3 + 4.1.2 jar true diff --git a/src/main/java/org/qortal/api/resource/BlocksResource.java b/src/main/java/org/qortal/api/resource/BlocksResource.java index 20207c70..ad5e466d 100644 --- a/src/main/java/org/qortal/api/resource/BlocksResource.java +++ b/src/main/java/org/qortal/api/resource/BlocksResource.java @@ -222,14 +222,25 @@ public class BlocksResource { } try (final Repository repository = RepositoryManager.getRepository()) { - // Check if the block exists in either the database or archive - if (repository.getBlockRepository().getHeightFromSignature(signature) == 0 && - repository.getBlockArchiveRepository().getHeightFromSignature(signature) == 0) { - // Not found in either the database or archive - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); - } + // Check if the block exists in either the database or archive + int height = repository.getBlockRepository().getHeightFromSignature(signature); + if (height == 0) { + height = repository.getBlockArchiveRepository().getHeightFromSignature(signature); + if (height == 0) { + // Not found in either the database or archive + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + } - return repository.getBlockRepository().getTransactionsFromSignature(signature, limit, offset, reverse); + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, height, height); + + // Expand signatures to transactions + List transactions = new ArrayList<>(signatures.size()); + for (byte[] s : signatures) { + transactions.add(repository.getTransactionRepository().fromSignature(s)); + } + + return transactions; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } diff --git a/src/main/java/org/qortal/api/resource/StatsResource.java b/src/main/java/org/qortal/api/resource/StatsResource.java new file mode 100644 index 00000000..c1588490 --- /dev/null +++ b/src/main/java/org/qortal/api/resource/StatsResource.java @@ -0,0 +1,70 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.api.*; +import org.qortal.block.BlockChain; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.Amounts; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import java.math.BigDecimal; +import java.util.List; + +@Path("/stats") +@Tag(name = "Stats") +public class StatsResource { + + private static final Logger LOGGER = LogManager.getLogger(StatsResource.class); + + + @Context + HttpServletRequest request; + + @GET + @Path("/supply/circulating") + @Operation( + summary = "Fetch circulating QORT supply", + responses = { + @ApiResponse( + description = "circulating supply of QORT", + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", format = "number")) + ) + } + ) + public BigDecimal circulatingSupply() { + long total = 0L; + + try (final Repository repository = RepositoryManager.getRepository()) { + int currentHeight = repository.getBlockRepository().getBlockchainHeight(); + + List rewardsByHeight = BlockChain.getInstance().getBlockRewardsByHeight(); + int rewardIndex = rewardsByHeight.size() - 1; + BlockChain.RewardByHeight rewardInfo = rewardsByHeight.get(rewardIndex); + + for (int height = currentHeight; height > 1; --height) { + if (height < rewardInfo.height) { + --rewardIndex; + rewardInfo = rewardsByHeight.get(rewardIndex); + } + + total += rewardInfo.reward; + } + + return Amounts.toBigDecimal(total); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + +} diff --git a/src/main/java/org/qortal/api/resource/TransactionsResource.java b/src/main/java/org/qortal/api/resource/TransactionsResource.java index 1311c4ad..61cef867 100644 --- a/src/main/java/org/qortal/api/resource/TransactionsResource.java +++ b/src/main/java/org/qortal/api/resource/TransactionsResource.java @@ -215,10 +215,25 @@ public class TransactionsResource { } try (final Repository repository = RepositoryManager.getRepository()) { - if (repository.getBlockRepository().getHeightFromSignature(signature) == 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + // Check if the block exists in either the database or archive + int height = repository.getBlockRepository().getHeightFromSignature(signature); + if (height == 0) { + height = repository.getBlockArchiveRepository().getHeightFromSignature(signature); + if (height == 0) { + // Not found in either the database or archive + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + } - return repository.getBlockRepository().getTransactionsFromSignature(signature, limit, offset, reverse); + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, height, height); + + // Expand signatures to transactions + List transactions = new ArrayList<>(signatures.size()); + for (byte[] s : signatures) { + transactions.add(repository.getTransactionRepository().fromSignature(s)); + } + + return transactions; } catch (ApiException e) { throw e; } catch (DataException e) { diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 540f8cf7..e030e6a6 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1686,12 +1686,14 @@ public class Block { transactionData.getSignature()); this.repository.getBlockRepository().save(blockTransactionData); - // Update transaction's height in repository + // Update transaction's height in repository and local transactionData transactionRepository.updateBlockHeight(transactionData.getSignature(), this.blockData.getHeight()); - - // Update local transactionData's height too transaction.getTransactionData().setBlockHeight(this.blockData.getHeight()); + // Update transaction's sequence in repository and local transactionData + transactionRepository.updateBlockSequence(transactionData.getSignature(), sequence); + transaction.getTransactionData().setBlockSequence(sequence); + // No longer unconfirmed transactionRepository.confirmTransaction(transactionData.getSignature()); @@ -1778,6 +1780,9 @@ public class Block { // Unset height transactionRepository.updateBlockHeight(transactionData.getSignature(), null); + + // Unset sequence + transactionRepository.updateBlockSequence(transactionData.getSignature(), null); } transactionRepository.deleteParticipants(transactionData); diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 88880887..218fb14d 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -871,6 +871,9 @@ public class BlockChain { BlockData orphanBlockData = repository.getBlockRepository().fromHeight(height); while (height > targetHeight) { + if (Controller.isStopping()) { + return false; + } LOGGER.info(String.format("Forcably orphaning block %d", height)); Block block = new Block(repository, orphanBlockData); diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 16373099..b9c3cde9 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -403,12 +403,12 @@ public class Controller extends Thread { RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); try (final Repository repository = RepositoryManager.getRepository()) { + RepositoryManager.rebuildTransactionSequences(repository); ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, false); } - } - catch (DataException e) { - // If exception has no cause then repository is in use by some other process. - if (e.getCause() == null) { + } catch (DataException e) { + // If exception has no cause or message then repository is in use by some other process. + if (e.getCause() == null && e.getMessage() == null) { LOGGER.info("Repository in use by another process?"); Gui.getInstance().fatalError("Repository issue", "Repository in use by another process?"); } else { @@ -442,6 +442,19 @@ public class Controller extends Thread { } } + try (Repository repository = RepositoryManager.getRepository()) { + if (RepositoryManager.needsTransactionSequenceRebuild(repository)) { + // Don't allow the node to start if transaction sequences haven't been built yet + // This is needed to handle a case when bootstrapping + LOGGER.error("Database upgrade needed. Please restart the core to complete the upgrade process."); + Gui.getInstance().fatalError("Database upgrade needed", "Please restart the core to complete the upgrade process."); + return; + } + } catch (DataException e) { + LOGGER.error("Error checking transaction sequences in repository", e); + return; + } + // Import current trade bot states and minting accounts if they exist Controller.importRepositoryData(); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index d3aadc43..f6b2dc0a 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -57,6 +57,8 @@ public class ArbitraryDataStorageManager extends Thread { * This must be higher than STORAGE_FULL_THRESHOLD in order to avoid a fetch/delete loop. */ public static final double DELETION_THRESHOLD = 0.98f; // 98% + private static final long PER_NAME_STORAGE_MULTIPLIER = 4L; + public ArbitraryDataStorageManager() { } @@ -535,7 +537,9 @@ public class ArbitraryDataStorageManager extends Thread { } double maxStorageCapacity = (double)this.storageCapacity * threshold; - long maxStoragePerName = (long)(maxStorageCapacity / (double)followedNamesCount); + + // Some names won't need/use much space, so give all names a 4x multiplier to compensate + long maxStoragePerName = (long)(maxStorageCapacity / (double)followedNamesCount) * PER_NAME_STORAGE_MULTIPLIER; return maxStoragePerName; } diff --git a/src/main/java/org/qortal/data/transaction/TransactionData.java b/src/main/java/org/qortal/data/transaction/TransactionData.java index 4bf3152c..c4a115df 100644 --- a/src/main/java/org/qortal/data/transaction/TransactionData.java +++ b/src/main/java/org/qortal/data/transaction/TransactionData.java @@ -79,6 +79,10 @@ public abstract class TransactionData { @Schema(accessMode = AccessMode.READ_ONLY, hidden = true, description = "height of block containing transaction") protected Integer blockHeight; + // Not always present + @Schema(accessMode = AccessMode.READ_ONLY, hidden = true, description = "sequence in block containing transaction") + protected Integer blockSequence; + // Not always present @Schema(accessMode = AccessMode.READ_ONLY, description = "group-approval status") protected ApprovalStatus approvalStatus; @@ -109,6 +113,7 @@ public abstract class TransactionData { this.fee = baseTransactionData.fee; this.signature = baseTransactionData.signature; this.blockHeight = baseTransactionData.blockHeight; + this.blockSequence = baseTransactionData.blockSequence; this.approvalStatus = baseTransactionData.approvalStatus; this.approvalHeight = baseTransactionData.approvalHeight; } @@ -177,6 +182,15 @@ public abstract class TransactionData { this.blockHeight = blockHeight; } + public Integer getBlockSequence() { + return this.blockSequence; + } + + @XmlTransient + public void setBlockSequence(Integer blockSequence) { + this.blockSequence = blockSequence; + } + public ApprovalStatus getApprovalStatus() { return approvalStatus; } diff --git a/src/main/java/org/qortal/network/Handshake.java b/src/main/java/org/qortal/network/Handshake.java index 47752767..4500cd59 100644 --- a/src/main/java/org/qortal/network/Handshake.java +++ b/src/main/java/org/qortal/network/Handshake.java @@ -265,7 +265,7 @@ public enum Handshake { private static final long PEER_VERSION_131 = 0x0100030001L; /** Minimum peer version that we are allowed to communicate with */ - private static final String MIN_PEER_VERSION = "3.8.2"; + private static final String MIN_PEER_VERSION = "4.0.0"; private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes private static final int POW_DIFFICULTY_PRE_131 = 8; // leading zero bits diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index fcf9e398..66b1d23e 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -2,18 +2,23 @@ package org.qortal.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.api.resource.TransactionsResource; -import org.qortal.controller.Controller; -import org.qortal.data.arbitrary.ArbitraryResourceData; -import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.block.Block; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.ATTransactionData; +import org.qortal.data.transaction.TransactionData; import org.qortal.gui.SplashFrame; import org.qortal.settings.Settings; -import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; +import org.qortal.transform.block.BlockTransformation; import java.sql.SQLException; -import java.util.List; +import java.util.*; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +import static org.qortal.transaction.Transaction.TransactionType.AT; public abstract class RepositoryManager { private static final Logger LOGGER = LogManager.getLogger(RepositoryManager.class); @@ -65,6 +70,164 @@ public abstract class RepositoryManager { } } + public static boolean needsTransactionSequenceRebuild(Repository repository) throws DataException { + // Check if we have any transactions without a block_sequence + List testSignatures = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria( + null, Arrays.asList("block_height IS NOT NULL AND block_sequence IS NULL"), new ArrayList<>(), 100); + if (testSignatures.isEmpty()) { + // block_sequence intact, so assume complete + return false; + } + + return true; + } + + public static boolean rebuildTransactionSequences(Repository repository) throws DataException { + if (Settings.getInstance().isLite()) { + // Lite nodes have no blockchain + return false; + } + if (Settings.getInstance().isTopOnly()) { + // topOnly nodes are unable to perform this reindex, and so are temporarily unsupported + throw new DataException("topOnly nodes are now unsupported, as they are missing data required for a db reshape"); + } + + try { + // Check if we have any unpopulated block_sequence values for the first 1000 blocks + if (!needsTransactionSequenceRebuild(repository)) { + // block_sequence already populated for the first 1000 blocks, so assume complete. + // We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so + // we shouldn't ever be left in a partially rebuilt state. + return false; + } + + LOGGER.info("Rebuilding transaction sequences - this will take a while..."); + + SplashFrame.getInstance().updateStatus("Rebuilding transactions - please wait..."); + + int blockchainHeight = repository.getBlockRepository().getBlockchainHeight(); + int totalTransactionCount = 0; + + for (int height = 1; height <= blockchainHeight; ++height) { + List inputTransactions = new ArrayList<>(); + + // Fetch block and transactions + BlockData blockData = repository.getBlockRepository().fromHeight(height); + boolean loadedFromArchive = false; + if (blockData == null) { + // Get (non-AT) transactions from the archive + BlockTransformation blockTransformation = BlockArchiveReader.getInstance().fetchBlockAtHeight(height); + blockData = blockTransformation.getBlockData(); + inputTransactions = blockTransformation.getTransactions(); // This doesn't include AT transactions + loadedFromArchive = true; + } + else { + // Get transactions from db + Block block = new Block(repository, blockData); + for (Transaction transaction : block.getTransactions()) { + inputTransactions.add(transaction.getTransactionData()); + } + } + + if (blockData == null) { + throw new DataException("Missing block data"); + } + + List transactions = new ArrayList<>(); + + if (loadedFromArchive) { + List transactionDataList = new ArrayList<>(blockData.getTransactionCount()); + // Fetch any AT transactions in this block + List atSignatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, height, height); + for (byte[] s : atSignatures) { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(s); + if (transactionData.getType() == AT) { + transactionDataList.add(transactionData); + } + } + + List atTransactions = new ArrayList<>(); + for (TransactionData transactionData : transactionDataList) { + ATTransactionData atTransactionData = (ATTransactionData) transactionData; + atTransactions.add(atTransactionData); + } + + // Create sorted list of ATs by creation time + List ats = new ArrayList<>(); + + for (ATTransactionData atTransactionData : atTransactions) { + ATData atData = repository.getATRepository().fromATAddress(atTransactionData.getATAddress()); + boolean hasExistingEntry = ats.stream().anyMatch(a -> Objects.equals(a.getATAddress(), atTransactionData.getATAddress())); + if (!hasExistingEntry) { + ats.add(atData); + } + } + + // Sort list of ATs by creation date + ats.sort(Comparator.comparingLong(ATData::getCreation)); + + // Loop through unique ATs + for (ATData atData : ats) { + List thisAtTransactions = atTransactions.stream() + .filter(t -> Objects.equals(t.getATAddress(), atData.getATAddress())) + .collect(Collectors.toList()); + + int count = thisAtTransactions.size(); + + if (count == 1) { + ATTransactionData atTransactionData = thisAtTransactions.get(0); + transactions.add(atTransactionData); + } + else if (count == 2) { + String atCreatorAddress = Crypto.toAddress(atData.getCreatorPublicKey()); + + ATTransactionData atTransactionData1 = thisAtTransactions.stream() + .filter(t -> !Objects.equals(t.getRecipient(), atCreatorAddress)) + .findFirst().orElse(null); + transactions.add(atTransactionData1); + + ATTransactionData atTransactionData2 = thisAtTransactions.stream() + .filter(t -> Objects.equals(t.getRecipient(), atCreatorAddress)) + .findFirst().orElse(null); + transactions.add(atTransactionData2); + } + else if (count > 2) { + LOGGER.info("Error: AT has more than 2 output transactions"); + } + } + } + + // Add all the regular transactions now that AT transactions have been handled + transactions.addAll(inputTransactions); + totalTransactionCount += transactions.size(); + + // Loop through and update sequences + for (int sequence = 0; sequence < transactions.size(); ++sequence) { + TransactionData transactionData = transactions.get(sequence); + + // Update transaction's sequence in repository + repository.getTransactionRepository().updateBlockSequence(transactionData.getSignature(), sequence); + } + + if (height % 10000 == 0) { + LOGGER.info("Rebuilt sequences for {} blocks (total transactions: {})", height, totalTransactionCount); + } + + repository.saveChanges(); + } + + LOGGER.info("Completed rebuild of transaction sequences."); + return true; + } + catch (DataException e) { + LOGGER.info("Unable to rebuild transaction sequences: {}. The database may have been left in an inconsistent state.", e.getMessage()); + + // Throw an exception so that the node startup is halted, allowing for a retry next time. + repository.discardChanges(); + throw new DataException("Rebuild of transaction sequences failed."); + } + } + public static void setRequestedCheckpoint(Boolean quick) { quickCheckpointRequested = quick; } diff --git a/src/main/java/org/qortal/repository/TransactionRepository.java b/src/main/java/org/qortal/repository/TransactionRepository.java index 105a317d..6cc88290 100644 --- a/src/main/java/org/qortal/repository/TransactionRepository.java +++ b/src/main/java/org/qortal/repository/TransactionRepository.java @@ -125,6 +125,23 @@ public interface TransactionRepository { public List getSignaturesMatchingCustomCriteria(TransactionType txType, List whereClauses, List bindParams) throws DataException; + /** + * Returns signatures for transactions that match search criteria, with optional limit. + *

+ * Alternate version that allows for custom where clauses and bind params. + * Only use for very specific use cases, such as the names integrity check. + * Not advised to be used otherwise, given that it could be possible for + * unsanitized inputs to be passed in if not careful. + * + * @param txType + * @param whereClauses + * @param bindParams + * @return + * @throws DataException + */ + public List getSignaturesMatchingCustomCriteria(TransactionType txType, List whereClauses, + List bindParams, Integer limit) throws DataException; + /** * Returns signature for latest auto-update transaction. *

@@ -309,6 +326,8 @@ public interface TransactionRepository { public void updateBlockHeight(byte[] signature, Integer height) throws DataException; + public void updateBlockSequence(byte[] signature, Integer sequence) throws DataException; + public void updateApprovalHeight(byte[] signature, Integer approvalHeight) throws DataException; /** diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index dd0404a8..33817309 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -296,10 +296,9 @@ public class HSQLDBATRepository implements ATRepository { @Override public Integer getATCreationBlockHeight(String atAddress) throws DataException { - String sql = "SELECT height " + String sql = "SELECT block_height " + "FROM DeployATTransactions " - + "JOIN BlockTransactions ON transaction_signature = signature " - + "JOIN Blocks ON Blocks.signature = block_signature " + + "JOIN Transactions USING (signature) " + "WHERE AT_address = ? " + "LIMIT 1"; @@ -877,18 +876,17 @@ public class HSQLDBATRepository implements ATRepository { public NextTransactionInfo findNextTransaction(String recipient, int height, int sequence) throws DataException { // We only need to search for a subset of transaction types: MESSAGE, PAYMENT or AT - String sql = "SELECT height, sequence, Transactions.signature " + String sql = "SELECT block_height, block_sequence, Transactions.signature " + "FROM (" + "SELECT signature FROM PaymentTransactions WHERE recipient = ? " + "UNION " + "SELECT signature FROM MessageTransactions WHERE recipient = ? " + "UNION " + "SELECT signature FROM ATTransactions WHERE recipient = ?" - + ") AS Transactions " - + "JOIN BlockTransactions ON BlockTransactions.transaction_signature = Transactions.signature " - + "JOIN Blocks ON Blocks.signature = BlockTransactions.block_signature " - + "WHERE (height > ? OR (height = ? AND sequence > ?)) " - + "ORDER BY height ASC, sequence ASC " + + ") AS SelectedTransactions " + + "JOIN Transactions USING (signature)" + + "WHERE (block_height > ? OR (block_height = ? AND block_sequence > ?)) " + + "ORDER BY block_height ASC, block_sequence ASC " + "LIMIT 1"; Object[] bindParams = new Object[] { recipient, recipient, recipient, height, height, sequence }; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index b54f4b08..cac02a9e 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -994,6 +994,17 @@ public class HSQLDBDatabaseUpdates { break; case 47: + // Add `block_sequence` to the Transaction table, as the BlockTransactions table is pruned for + // older blocks and therefore the sequence becomes unavailable + LOGGER.info("Reshaping Transactions table - this can take a while..."); + stmt.execute("ALTER TABLE Transactions ADD block_sequence INTEGER"); + + // For finding transactions by height and sequence + LOGGER.info("Adding index to Transactions table - this can take a while..."); + stmt.execute("CREATE INDEX TransactionHeightSequenceIndex on Transactions (block_height, block_sequence)"); + break; + + case 48: // We need to keep a local cache of arbitrary resources (items published to QDN), for easier searching. // IMPORTANT: this is a cache of the last known state of a resource (both confirmed // and valid unconfirmed). It cannot be assumed that all nodes will contain the same state at a diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index a8df1ab5..740b3e65 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -194,8 +194,7 @@ public class HSQLDBTransactionRepository implements TransactionRepository { @Override public TransactionData fromHeightAndSequence(int height, int sequence) throws DataException { - String sql = "SELECT transaction_signature FROM BlockTransactions JOIN Blocks ON signature = block_signature " - + "WHERE height = ? AND sequence = ?"; + String sql = "SELECT signature FROM Transactions WHERE block_height = ? AND block_sequence = ?"; try (ResultSet resultSet = this.repository.checkedExecute(sql, height, sequence)) { if (resultSet == null) @@ -657,8 +656,13 @@ public class HSQLDBTransactionRepository implements TransactionRepository { List bindParams) throws DataException { List signatures = new ArrayList<>(); + String txTypeClassName = ""; + if (txType != null) { + txTypeClassName = txType.className; + } + StringBuilder sql = new StringBuilder(1024); - sql.append(String.format("SELECT signature FROM %sTransactions", txType.className)); + sql.append(String.format("SELECT signature FROM %sTransactions", txTypeClassName)); if (!whereClauses.isEmpty()) { sql.append(" WHERE "); @@ -690,6 +694,53 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } } + public List getSignaturesMatchingCustomCriteria(TransactionType txType, List whereClauses, + List bindParams, Integer limit) throws DataException { + List signatures = new ArrayList<>(); + + String txTypeClassName = ""; + if (txType != null) { + txTypeClassName = txType.className; + } + + StringBuilder sql = new StringBuilder(1024); + sql.append(String.format("SELECT signature FROM %sTransactions", txTypeClassName)); + + if (!whereClauses.isEmpty()) { + sql.append(" WHERE "); + + final int whereClausesSize = whereClauses.size(); + for (int wci = 0; wci < whereClausesSize; ++wci) { + if (wci != 0) + sql.append(" AND "); + + sql.append(whereClauses.get(wci)); + } + } + + if (limit != null) { + sql.append(" LIMIT ?"); + bindParams.add(limit); + } + + LOGGER.trace(() -> String.format("Transaction search SQL: %s", sql)); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return signatures; + + do { + byte[] signature = resultSet.getBytes(1); + + signatures.add(signature); + } while (resultSet.next()); + + return signatures; + } catch (SQLException e) { + throw new DataException("Unable to fetch matching transaction signatures from repository", e); + } + } + @Override public byte[] getLatestAutoUpdateTransaction(TransactionType txType, int txGroupId, Integer service) throws DataException { StringBuilder sql = new StringBuilder(1024); @@ -1444,6 +1495,19 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } } + @Override + public void updateBlockSequence(byte[] signature, Integer blockSequence) throws DataException { + HSQLDBSaver saver = new HSQLDBSaver("Transactions"); + + saver.bind("signature", signature).bind("block_sequence", blockSequence); + + try { + saver.execute(repository); + } catch (SQLException e) { + throw new DataException("Unable to update transaction's block sequence in repository", e); + } + } + @Override public void updateApprovalHeight(byte[] signature, Integer approvalHeight) throws DataException { HSQLDBSaver saver = new HSQLDBSaver("Transactions"); diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index a87a72f4..362227a5 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -181,7 +181,7 @@ public class Settings { /** How often to attempt archiving (ms). */ private long archiveInterval = 7171L; // milliseconds /** Serialization version to use when building an archive */ - private int defaultArchiveVersion = 1; + private int defaultArchiveVersion = 2; /** Whether to automatically bootstrap instead of syncing from genesis */ @@ -201,25 +201,25 @@ public class Settings { /** Whether to attempt to open the listen port via UPnP */ private boolean uPnPEnabled = true; /** Minimum number of peers to allow block minting / synchronization. */ - private int minBlockchainPeers = 5; + private int minBlockchainPeers = 3; /** Target number of outbound connections to peers we should make. */ private int minOutboundPeers = 16; /** Maximum number of peer connections we allow. */ - private int maxPeers = 36; + private int maxPeers = 40; /** Number of slots to reserve for short-lived QDN data transfers */ private int maxDataPeers = 4; /** Maximum number of threads for network engine. */ - private int maxNetworkThreadPoolSize = 32; + private int maxNetworkThreadPoolSize = 120; /** Maximum number of threads for network proof-of-work compute, used during handshaking. */ private int networkPoWComputePoolSize = 2; /** Maximum number of retry attempts if a peer fails to respond with the requested data */ private int maxRetries = 2; /** The number of seconds of no activity before recovery mode begins */ - public long recoveryModeTimeout = 10 * 60 * 1000L; + public long recoveryModeTimeout = 24 * 60 * 60 * 1000L; /** Minimum peer version number required in order to sync with them */ - private String minPeerVersion = "3.8.7"; + private String minPeerVersion = "4.1.1"; /** Whether to allow connections with peers below minPeerVersion * If true, we won't sync with them but they can still sync with us, and will show in the peers list * If false, sync will be blocked both ways, and they will not appear in the peers list */ @@ -267,7 +267,7 @@ public class Settings { /** Repository storage path. */ private String repositoryPath = "db"; /** Repository connection pool size. Needs to be a bit bigger than maxNetworkThreadPoolSize */ - private int repositoryConnectionPoolSize = 100; + private int repositoryConnectionPoolSize = 240; private List fixedNetwork; // Export/import @@ -508,6 +508,9 @@ public class Settings { if (this.minBlockchainPeers < 1 && !singleNodeTestnet) throwValidationError("minBlockchainPeers must be at least 1"); + if (this.topOnly) + throwValidationError("topOnly mode is no longer supported"); + if (this.apiKey != null && this.apiKey.trim().length() < 8) throwValidationError("apiKey must be at least 8 characters"); diff --git a/src/main/java/org/qortal/utils/BlockArchiveUtils.java b/src/main/java/org/qortal/utils/BlockArchiveUtils.java index 807faef9..f9ca0d0d 100644 --- a/src/main/java/org/qortal/utils/BlockArchiveUtils.java +++ b/src/main/java/org/qortal/utils/BlockArchiveUtils.java @@ -1,5 +1,7 @@ package org.qortal.utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.qortal.data.at.ATStateData; import org.qortal.data.block.BlockData; import org.qortal.data.transaction.TransactionData; @@ -12,6 +14,8 @@ import java.util.List; public class BlockArchiveUtils { + private static final Logger LOGGER = LogManager.getLogger(BlockArchiveUtils.class); + /** * importFromArchive *

@@ -87,7 +91,8 @@ public class BlockArchiveUtils { } catch (DataException e) { repository.discardChanges(); - throw new IllegalStateException("Unable to import blocks from archive"); + LOGGER.info("Unable to import blocks from archive", e); + throw(e); } } repository.saveChanges(); diff --git a/src/test/java/org/qortal/test/BootstrapTests.java b/src/test/java/org/qortal/test/BootstrapTests.java index b60b412c..58e1cfa2 100644 --- a/src/test/java/org/qortal/test/BootstrapTests.java +++ b/src/test/java/org/qortal/test/BootstrapTests.java @@ -212,7 +212,7 @@ public class BootstrapTests extends Common { @Test public void testBootstrapHosts() throws IOException { String[] bootstrapHosts = Settings.getInstance().getBootstrapHosts(); - String[] bootstrapTypes = { "archive", "toponly" }; + String[] bootstrapTypes = { "archive" }; // , "toponly" for (String host : bootstrapHosts) { for (String type : bootstrapTypes) { diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java index 028c054d..c05ceabf 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java @@ -113,13 +113,16 @@ public class ArbitraryDataStorageCapacityTests extends Common { assertTrue(resourceListManager.addToList("followedNames", "Test2", false)); assertTrue(resourceListManager.addToList("followedNames", "Test3", false)); assertTrue(resourceListManager.addToList("followedNames", "Test4", false)); + assertTrue(resourceListManager.addToList("followedNames", "Test5", false)); + assertTrue(resourceListManager.addToList("followedNames", "Test6", false)); // Ensure the followed name count is correct - assertEquals(4, resourceListManager.getItemCountForList("followedNames")); - assertEquals(4, ListUtils.followedNamesCount()); + assertEquals(6, resourceListManager.getItemCountForList("followedNames")); + assertEquals(6, ListUtils.followedNamesCount()); // Storage space per name should be the total storage capacity divided by the number of names - long expectedStorageCapacityPerName = (long)(totalStorageCapacity / 4.0f); + // then multiplied by 4, to allow for names that don't use much space + long expectedStorageCapacityPerName = (long)(totalStorageCapacity / 6.0f) * 4L; assertEquals(expectedStorageCapacityPerName, storageManager.storageCapacityPerName(storageFullThreshold)); } diff --git a/src/test/java/org/qortal/test/crosschain/BitcoinTests.java b/src/test/java/org/qortal/test/crosschain/BitcoinTests.java index 07a01ce2..1096d7ad 100644 --- a/src/test/java/org/qortal/test/crosschain/BitcoinTests.java +++ b/src/test/java/org/qortal/test/crosschain/BitcoinTests.java @@ -8,6 +8,7 @@ import org.bitcoinj.core.Transaction; import org.bitcoinj.store.BlockStoreException; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.qortal.crosschain.Bitcoin; import org.qortal.crosschain.ForeignBlockchainException; @@ -32,6 +33,7 @@ public class BitcoinTests extends Common { } @Test + @Ignore("Often fails due to unreliable BTC testnet ElectrumX servers") public void testGetMedianBlockTime() throws BlockStoreException, ForeignBlockchainException { System.out.println(String.format("Starting BTC instance...")); System.out.println(String.format("BTC instance started")); @@ -53,6 +55,7 @@ public class BitcoinTests extends Common { } @Test + @Ignore("Often fails due to unreliable BTC testnet ElectrumX servers") public void testFindHtlcSecret() throws ForeignBlockchainException { // This actually exists on TEST3 but can take a while to fetch String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; @@ -65,6 +68,7 @@ public class BitcoinTests extends Common { } @Test + @Ignore("Often fails due to unreliable BTC testnet ElectrumX servers") public void testBuildSpend() { String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; @@ -81,6 +85,7 @@ public class BitcoinTests extends Common { } @Test + @Ignore("Often fails due to unreliable BTC testnet ElectrumX servers") public void testGetWalletBalance() throws ForeignBlockchainException { String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; @@ -102,6 +107,7 @@ public class BitcoinTests extends Common { } @Test + @Ignore("Often fails due to unreliable BTC testnet ElectrumX servers") public void testGetUnusedReceiveAddress() throws ForeignBlockchainException { String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; diff --git a/src/test/java/org/qortal/test/crosschain/HtlcTests.java b/src/test/java/org/qortal/test/crosschain/HtlcTests.java index 75b290bf..3f3678f7 100644 --- a/src/test/java/org/qortal/test/crosschain/HtlcTests.java +++ b/src/test/java/org/qortal/test/crosschain/HtlcTests.java @@ -8,6 +8,7 @@ import org.junit.Ignore; import org.junit.Test; import org.qortal.crosschain.Bitcoin; import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Litecoin; import org.qortal.crypto.Crypto; import org.qortal.crosschain.BitcoinyHTLC; import org.qortal.repository.DataException; @@ -18,17 +19,19 @@ import com.google.common.primitives.Longs; public class HtlcTests extends Common { private Bitcoin bitcoin; + private Litecoin litecoin; @Before public void beforeTest() throws DataException { Common.useDefaultSettings(); // TestNet3 bitcoin = Bitcoin.getInstance(); + litecoin = Litecoin.getInstance(); } @After public void afterTest() { Bitcoin.resetForTesting(); - bitcoin = null; + litecoin = null; } @Test @@ -52,12 +55,12 @@ public class HtlcTests extends Common { do { // We need to perform fresh setup for 1st test Bitcoin.resetForTesting(); - bitcoin = Bitcoin.getInstance(); + litecoin = Litecoin.getInstance(); long now = System.currentTimeMillis(); long timestampBoundary = now / 30_000L; - byte[] secret1 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); + byte[] secret1 = BitcoinyHTLC.findHtlcSecret(litecoin, p2shAddress); long executionPeriod1 = System.currentTimeMillis() - now; assertNotNull(secret1); @@ -65,7 +68,7 @@ public class HtlcTests extends Common { assertTrue("1st execution period should not be instant!", executionPeriod1 > 10); - byte[] secret2 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); + byte[] secret2 = BitcoinyHTLC.findHtlcSecret(litecoin, p2shAddress); long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1; assertNotNull(secret2); @@ -86,7 +89,7 @@ public class HtlcTests extends Common { // This actually exists on TEST3 but can take a while to fetch String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; - BitcoinyHTLC.Status htlcStatus = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L); + BitcoinyHTLC.Status htlcStatus = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddress, 1L); assertNotNull(htlcStatus); System.out.println(String.format("HTLC %s status: %s", p2shAddress, htlcStatus.name())); @@ -97,21 +100,21 @@ public class HtlcTests extends Common { do { // We need to perform fresh setup for 1st test Bitcoin.resetForTesting(); - bitcoin = Bitcoin.getInstance(); + litecoin = Litecoin.getInstance(); long now = System.currentTimeMillis(); long timestampBoundary = now / 30_000L; // Won't ever exist - String p2shAddress = bitcoin.deriveP2shAddress(Crypto.hash160(Longs.toByteArray(now))); + String p2shAddress = litecoin.deriveP2shAddress(Crypto.hash160(Longs.toByteArray(now))); - BitcoinyHTLC.Status htlcStatus1 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L); + BitcoinyHTLC.Status htlcStatus1 = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddress, 1L); long executionPeriod1 = System.currentTimeMillis() - now; assertNotNull(htlcStatus1); assertTrue("1st execution period should not be instant!", executionPeriod1 > 10); - BitcoinyHTLC.Status htlcStatus2 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L); + BitcoinyHTLC.Status htlcStatus2 = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddress, 1L); long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1; assertNotNull(htlcStatus2); diff --git a/src/test/java/org/qortal/test/crosschain/LitecoinTests.java b/src/test/java/org/qortal/test/crosschain/LitecoinTests.java index 6236483a..5ea7bc95 100644 --- a/src/test/java/org/qortal/test/crosschain/LitecoinTests.java +++ b/src/test/java/org/qortal/test/crosschain/LitecoinTests.java @@ -5,7 +5,6 @@ import static org.junit.Assert.*; import java.util.Arrays; import org.bitcoinj.core.Transaction; -import org.bitcoinj.store.BlockStoreException; import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -33,12 +32,12 @@ public class LitecoinTests extends Common { } @Test - public void testGetMedianBlockTime() throws BlockStoreException, ForeignBlockchainException { + public void testGetMedianBlockTime() throws ForeignBlockchainException { long before = System.currentTimeMillis(); - System.out.println(String.format("Bitcoin median blocktime: %d", litecoin.getMedianBlockTime())); + System.out.println(String.format("Litecoin median blocktime: %d", litecoin.getMedianBlockTime())); long afterFirst = System.currentTimeMillis(); - System.out.println(String.format("Bitcoin median blocktime: %d", litecoin.getMedianBlockTime())); + System.out.println(String.format("Litecoin median blocktime: %d", litecoin.getMedianBlockTime())); long afterSecond = System.currentTimeMillis(); long firstPeriod = afterFirst - before; diff --git a/start.sh b/start.sh index b3db54fe..a7a1419f 100755 --- a/start.sh +++ b/start.sh @@ -33,7 +33,8 @@ fi # Limits Java JVM stack size and maximum heap usage. # Comment out for bigger systems, e.g. non-routers # or when API documentation is enabled -# JVM_MEMORY_ARGS="-Xss256k -Xmx128m" +# Uncomment (remove '#' sign) line below if your system has less than 12GB of RAM for optimal RAM defaults +# JVM_MEMORY_ARGS="-Xss1256k -Xmx3128m" # Although java.net.preferIPv4Stack is supposed to be false # by default in Java 11, on some platforms (e.g. FreeBSD 12),