diff --git a/WindowsInstaller/Qortal.aip b/WindowsInstaller/Qortal.aip index 722d881e..e922943d 100755 --- a/WindowsInstaller/Qortal.aip +++ b/WindowsInstaller/Qortal.aip @@ -17,10 +17,10 @@ - + - + @@ -212,7 +212,7 @@ - + diff --git a/src/main/java/org/qortal/account/Account.java b/src/main/java/org/qortal/account/Account.java index df14d88f..c3a25fb6 100644 --- a/src/main/java/org/qortal/account/Account.java +++ b/src/main/java/org/qortal/account/Account.java @@ -8,11 +8,13 @@ import javax.xml.bind.annotation.XmlAccessorType; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.block.BlockChain; +import org.qortal.controller.LiteNode; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; import org.qortal.data.account.RewardShareData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; +import org.qortal.settings.Settings; import org.qortal.utils.Base58; @XmlAccessorType(XmlAccessType.NONE) // Stops JAX-RS errors when unmarshalling blockchain config @@ -59,7 +61,17 @@ public class Account { // Balance manipulations - assetId is 0 for QORT public long getConfirmedBalance(long assetId) throws DataException { - AccountBalanceData accountBalanceData = this.repository.getAccountRepository().getBalance(this.address, assetId); + AccountBalanceData accountBalanceData; + + if (Settings.getInstance().isLite()) { + // Lite nodes request data from peers instead of the local db + accountBalanceData = LiteNode.getInstance().fetchAccountBalance(this.address, assetId); + } + else { + // All other node types fetch from the local db + accountBalanceData = this.repository.getAccountRepository().getBalance(this.address, assetId); + } + if (accountBalanceData == null) return 0; diff --git a/src/main/java/org/qortal/api/model/NodeInfo.java b/src/main/java/org/qortal/api/model/NodeInfo.java index 16a4df75..6732357a 100644 --- a/src/main/java/org/qortal/api/model/NodeInfo.java +++ b/src/main/java/org/qortal/api/model/NodeInfo.java @@ -12,6 +12,7 @@ public class NodeInfo { public long buildTimestamp; public String nodeId; public boolean isTestNet; + public String type; public NodeInfo() { } diff --git a/src/main/java/org/qortal/api/resource/AddressesResource.java b/src/main/java/org/qortal/api/resource/AddressesResource.java index b5268db7..4de8d908 100644 --- a/src/main/java/org/qortal/api/resource/AddressesResource.java +++ b/src/main/java/org/qortal/api/resource/AddressesResource.java @@ -30,6 +30,7 @@ import org.qortal.api.Security; import org.qortal.api.model.ApiOnlineAccount; import org.qortal.api.model.RewardShareKeyRequest; import org.qortal.asset.Asset; +import org.qortal.controller.LiteNode; import org.qortal.controller.OnlineAccountsManager; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountData; @@ -109,18 +110,26 @@ public class AddressesResource { if (!Crypto.isValidAddress(address)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - byte[] lastReference = null; + AccountData accountData; - try (final Repository repository = RepositoryManager.getRepository()) { - AccountData accountData = repository.getAccountRepository().getAccount(address); - // Not found? - if (accountData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - - lastReference = accountData.getReference(); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + if (Settings.getInstance().isLite()) { + // Lite nodes request data from peers instead of the local db + accountData = LiteNode.getInstance().fetchAccountData(address); } + else { + // All other node types request data from local db + try (final Repository repository = RepositoryManager.getRepository()) { + accountData = repository.getAccountRepository().getAccount(address); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + // Not found? + if (accountData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + byte[] lastReference = accountData.getReference(); if (lastReference == null || lastReference.length == 0) return "false"; diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index efb47acf..bf7294ab 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -119,10 +119,23 @@ public class AdminResource { nodeInfo.buildTimestamp = Controller.getInstance().getBuildTimestamp(); nodeInfo.nodeId = Network.getInstance().getOurNodeId(); nodeInfo.isTestNet = Settings.getInstance().isTestNet(); + nodeInfo.type = getNodeType(); return nodeInfo; } + private String getNodeType() { + if (Settings.getInstance().isTopOnly()) { + return "topOnly"; + } + else if (Settings.getInstance().isLite()) { + return "lite"; + } + else { + return "full"; + } + } + @GET @Path("/status") @Operation( diff --git a/src/main/java/org/qortal/api/resource/NamesResource.java b/src/main/java/org/qortal/api/resource/NamesResource.java index e380ab55..a900d6bf 100644 --- a/src/main/java/org/qortal/api/resource/NamesResource.java +++ b/src/main/java/org/qortal/api/resource/NamesResource.java @@ -26,6 +26,7 @@ import org.qortal.api.ApiErrors; import org.qortal.api.ApiException; import org.qortal.api.ApiExceptionFactory; import org.qortal.api.model.NameSummary; +import org.qortal.controller.LiteNode; import org.qortal.crypto.Crypto; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.BuyNameTransactionData; @@ -101,7 +102,14 @@ public class NamesResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); try (final Repository repository = RepositoryManager.getRepository()) { - List names = repository.getNameRepository().getNamesByOwner(address, limit, offset, reverse); + List names; + + if (Settings.getInstance().isLite()) { + names = LiteNode.getInstance().fetchAccountNames(address); + } + else { + names = repository.getNameRepository().getNamesByOwner(address, limit, offset, reverse); + } return names.stream().map(NameSummary::new).collect(Collectors.toList()); } catch (DataException e) { @@ -126,10 +134,18 @@ public class NamesResource { @ApiErrors({ApiError.NAME_UNKNOWN, ApiError.REPOSITORY_ISSUE}) public NameData getName(@PathParam("name") String name) { try (final Repository repository = RepositoryManager.getRepository()) { - NameData nameData = repository.getNameRepository().fromName(name); + NameData nameData; - if (nameData == null) + if (Settings.getInstance().isLite()) { + nameData = LiteNode.getInstance().fetchNameData(name); + } + else { + nameData = repository.getNameRepository().fromName(name); + } + + if (nameData == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NAME_UNKNOWN); + } return nameData; } catch (ApiException e) { diff --git a/src/main/java/org/qortal/api/resource/TransactionsResource.java b/src/main/java/org/qortal/api/resource/TransactionsResource.java index 55ad7cde..4c440304 100644 --- a/src/main/java/org/qortal/api/resource/TransactionsResource.java +++ b/src/main/java/org/qortal/api/resource/TransactionsResource.java @@ -12,6 +12,7 @@ import io.swagger.v3.oas.annotations.tags.Tag; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; @@ -32,6 +33,8 @@ import org.qortal.api.ApiException; import org.qortal.api.ApiExceptionFactory; import org.qortal.api.model.SimpleTransactionSignRequest; import org.qortal.controller.Controller; +import org.qortal.controller.LiteNode; +import org.qortal.crypto.Crypto; import org.qortal.data.transaction.TransactionData; import org.qortal.globalization.Translator; import org.qortal.repository.DataException; @@ -250,14 +253,29 @@ public class TransactionsResource { ApiError.REPOSITORY_ISSUE }) public List getUnconfirmedTransactions(@Parameter( + description = "A list of transaction types" + ) @QueryParam("txType") List txTypes, @Parameter( + description = "Transaction creator's base58 encoded public key" + ) @QueryParam("creator") String creatorPublicKey58, @Parameter( ref = "limit" ) @QueryParam("limit") Integer limit, @Parameter( ref = "offset" ) @QueryParam("offset") Integer offset, @Parameter( ref = "reverse" ) @QueryParam("reverse") Boolean reverse) { + + // Decode public key if supplied + byte[] creatorPublicKey = null; + if (creatorPublicKey58 != null) { + try { + creatorPublicKey = Base58.decode(creatorPublicKey58); + } catch (NumberFormatException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY, e); + } + } + try (final Repository repository = RepositoryManager.getRepository()) { - return repository.getTransactionRepository().getUnconfirmedTransactions(limit, offset, reverse); + return repository.getTransactionRepository().getUnconfirmedTransactions(txTypes, creatorPublicKey, limit, offset, reverse); } catch (ApiException e) { throw e; } catch (DataException e) { @@ -366,6 +384,73 @@ public class TransactionsResource { } } + @GET + @Path("/address/{address}") + @Operation( + summary = "Returns transactions for given address", + responses = { + @ApiResponse( + description = "transactions", + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = TransactionData.class + ) + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE}) + public List getAddressTransactions(@PathParam("address") String address, + @Parameter(ref = "limit") @QueryParam("limit") Integer limit, + @Parameter(ref = "offset") @QueryParam("offset") Integer offset, + @Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) { + if (!Crypto.isValidAddress(address)) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + } + + if (limit == null) { + limit = 0; + } + if (offset == null) { + offset = 0; + } + + List transactions; + + if (Settings.getInstance().isLite()) { + // Fetch from network + transactions = LiteNode.getInstance().fetchAccountTransactions(address, limit, offset); + + // Sort the data, since we can't guarantee the order that a peer sent it in + if (reverse) { + transactions.sort(Comparator.comparingLong(TransactionData::getTimestamp).reversed()); + } else { + transactions.sort(Comparator.comparingLong(TransactionData::getTimestamp)); + } + } + else { + // Fetch from local db + try (final Repository repository = RepositoryManager.getRepository()) { + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, + null, null, null, address, TransactionsResource.ConfirmationStatus.CONFIRMED, limit, offset, reverse); + + // Expand signatures to transactions + transactions = new ArrayList<>(signatures.size()); + for (byte[] signature : signatures) { + transactions.add(repository.getTransactionRepository().fromSignature(signature)); + } + } catch (ApiException e) { + throw e; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + return transactions; + } + @GET @Path("/unitfee") @Operation( diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index bc06fadf..44ad4a7f 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -69,7 +69,8 @@ public class BlockChain { newBlockSigHeight, shareBinFix, calcChainWeightTimestamp, - transactionV5Timestamp; + transactionV5Timestamp, + transactionV6Timestamp; } // Custom transaction fees @@ -405,6 +406,10 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.transactionV5Timestamp.name()).longValue(); } + public long getTransactionV6Timestamp() { + return this.featureTriggers.get(FeatureTrigger.transactionV6Timestamp.name()).longValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 04797314..9966d6a9 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -61,6 +61,11 @@ public class BlockMinter extends Thread { public void run() { Thread.currentThread().setName("BlockMinter"); + if (Settings.getInstance().isLite()) { + // Lite nodes do not mint + return; + } + try (final Repository repository = RepositoryManager.getRepository()) { if (Settings.getInstance().getWipeUnconfirmedOnStart()) { // Wipe existing unconfirmed transactions diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index f902e1b0..3d6da8bf 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -32,6 +32,7 @@ import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; import org.qortal.api.ApiService; import org.qortal.api.DomainMapService; import org.qortal.api.GatewayService; +import org.qortal.api.resource.TransactionsResource; import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; @@ -39,8 +40,11 @@ import org.qortal.controller.arbitrary.*; import org.qortal.controller.repository.PruneManager; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.controller.tradebot.TradeBot; +import org.qortal.data.account.AccountBalanceData; +import org.qortal.data.account.AccountData; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; +import org.qortal.data.naming.NameData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; import org.qortal.data.transaction.ChatTransactionData; @@ -179,6 +183,52 @@ public class Controller extends Thread { } public GetArbitraryMetadataMessageStats getArbitraryMetadataMessageStats = new GetArbitraryMetadataMessageStats(); + public static class GetAccountMessageStats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong cacheHits = new AtomicLong(); + public AtomicLong unknownAccounts = new AtomicLong(); + + public GetAccountMessageStats() { + } + } + public GetAccountMessageStats getAccountMessageStats = new GetAccountMessageStats(); + + public static class GetAccountBalanceMessageStats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong unknownAccounts = new AtomicLong(); + + public GetAccountBalanceMessageStats() { + } + } + public GetAccountBalanceMessageStats getAccountBalanceMessageStats = new GetAccountBalanceMessageStats(); + + public static class GetAccountTransactionsMessageStats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong unknownAccounts = new AtomicLong(); + + public GetAccountTransactionsMessageStats() { + } + } + public GetAccountTransactionsMessageStats getAccountTransactionsMessageStats = new GetAccountTransactionsMessageStats(); + + public static class GetAccountNamesMessageStats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong unknownAccounts = new AtomicLong(); + + public GetAccountNamesMessageStats() { + } + } + public GetAccountNamesMessageStats getAccountNamesMessageStats = new GetAccountNamesMessageStats(); + + public static class GetNameMessageStats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong unknownAccounts = new AtomicLong(); + + public GetNameMessageStats() { + } + } + public GetNameMessageStats getNameMessageStats = new GetNameMessageStats(); + public AtomicLong latestBlocksCacheRefills = new AtomicLong(); public StatsSnapshot() { @@ -363,23 +413,27 @@ public class Controller extends Thread { return; // Not System.exit() so that GUI can display error } - // Rebuild Names table and check database integrity (if enabled) - NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck(); - namesDatabaseIntegrityCheck.rebuildAllNames(); - if (Settings.getInstance().isNamesIntegrityCheckEnabled()) { - namesDatabaseIntegrityCheck.runIntegrityCheck(); - } + // If we have a non-lite node, we need to perform some startup actions + if (!Settings.getInstance().isLite()) { - LOGGER.info("Validating blockchain"); - try { - BlockChain.validate(); + // Rebuild Names table and check database integrity (if enabled) + NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck(); + namesDatabaseIntegrityCheck.rebuildAllNames(); + if (Settings.getInstance().isNamesIntegrityCheckEnabled()) { + namesDatabaseIntegrityCheck.runIntegrityCheck(); + } - Controller.getInstance().refillLatestBlocksCache(); - LOGGER.info(String.format("Our chain height at start-up: %d", Controller.getInstance().getChainHeight())); - } catch (DataException e) { - LOGGER.error("Couldn't validate blockchain", e); - Gui.getInstance().fatalError("Blockchain validation issue", e); - return; // Not System.exit() so that GUI can display error + LOGGER.info("Validating blockchain"); + try { + BlockChain.validate(); + + Controller.getInstance().refillLatestBlocksCache(); + LOGGER.info(String.format("Our chain height at start-up: %d", Controller.getInstance().getChainHeight())); + } catch (DataException e) { + LOGGER.error("Couldn't validate blockchain", e); + Gui.getInstance().fatalError("Blockchain validation issue", e); + return; // Not System.exit() so that GUI can display error + } } // Import current trade bot states and minting accounts if they exist @@ -740,7 +794,11 @@ public class Controller extends Thread { final Long minLatestBlockTimestamp = NTP.getTime() - (30 * 60 * 1000L); synchronized (Synchronizer.getInstance().syncLock) { - if (this.isMintingPossible) { + if (Settings.getInstance().isLite()) { + actionText = Translator.INSTANCE.translate("SysTray", "LITE_NODE"); + SysTray.getInstance().setTrayIcon(4); + } + else if (this.isMintingPossible) { actionText = Translator.INSTANCE.translate("SysTray", "MINTING_ENABLED"); SysTray.getInstance().setTrayIcon(2); } @@ -762,7 +820,11 @@ public class Controller extends Thread { } } - String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height) + "\n" + String.format("%s: %s", Translator.INSTANCE.translate("SysTray", "BUILD_VERSION"), this.buildVersion); + String tooltip = String.format("%s - %d %s", actionText, numberOfPeers, connectionsText); + if (!Settings.getInstance().isLite()) { + tooltip = tooltip.concat(String.format(" - %s %d", heightText, height)); + } + tooltip = tooltip.concat(String.format("\n%s: %s", Translator.INSTANCE.translate("SysTray", "BUILD_VERSION"), this.buildVersion)); SysTray.getInstance().setToolTipText(tooltip); this.callbackExecutor.execute(() -> { @@ -922,6 +984,11 @@ public class Controller extends Thread { // Callbacks for/from network public void doNetworkBroadcast() { + if (Settings.getInstance().isLite()) { + // Lite nodes have nothing to broadcast + return; + } + Network network = Network.getInstance(); // Send (if outbound) / Request peer lists @@ -1204,6 +1271,26 @@ public class Controller extends Thread { TradeBot.getInstance().onTradePresencesMessage(peer, message); break; + case GET_ACCOUNT: + onNetworkGetAccountMessage(peer, message); + break; + + case GET_ACCOUNT_BALANCE: + onNetworkGetAccountBalanceMessage(peer, message); + break; + + case GET_ACCOUNT_TRANSACTIONS: + onNetworkGetAccountTransactionsMessage(peer, message); + break; + + case GET_ACCOUNT_NAMES: + onNetworkGetAccountNamesMessage(peer, message); + break; + + case GET_NAME: + onNetworkGetNameMessage(peer, message); + break; + default: LOGGER.debug(() -> String.format("Unhandled %s message [ID %d] from peer %s", message.getType().name(), message.getId(), peer)); break; @@ -1440,11 +1527,13 @@ public class Controller extends Thread { private void onNetworkHeightV2Message(Peer peer, Message message) { HeightV2Message heightV2Message = (HeightV2Message) message; - // If peer is inbound and we've not updated their height - // then this is probably their initial HEIGHT_V2 message - // so they need a corresponding HEIGHT_V2 message from us - if (!peer.isOutbound() && (peer.getChainTipData() == null || peer.getChainTipData().getLastHeight() == null)) - peer.sendMessage(Network.getInstance().buildHeightMessage(peer, getChainTip())); + if (!Settings.getInstance().isLite()) { + // If peer is inbound and we've not updated their height + // then this is probably their initial HEIGHT_V2 message + // so they need a corresponding HEIGHT_V2 message from us + if (!peer.isOutbound() && (peer.getChainTipData() == null || peer.getChainTipData().getLastHeight() == null)) + peer.sendMessage(Network.getInstance().buildHeightMessage(peer, getChainTip())); + } // Update peer chain tip data PeerChainTipData newChainTipData = new PeerChainTipData(heightV2Message.getHeight(), heightV2Message.getSignature(), heightV2Message.getTimestamp(), heightV2Message.getMinterPublicKey()); @@ -1454,6 +1543,193 @@ public class Controller extends Thread { Synchronizer.getInstance().requestSync(); } + private void onNetworkGetAccountMessage(Peer peer, Message message) { + GetAccountMessage getAccountMessage = (GetAccountMessage) message; + String address = getAccountMessage.getAddress(); + this.stats.getAccountMessageStats.requests.incrementAndGet(); + + try (final Repository repository = RepositoryManager.getRepository()) { + AccountData accountData = repository.getAccountRepository().getAccount(address); + + if (accountData == null) { + // We don't have this account + this.stats.getAccountMessageStats.unknownAccounts.getAndIncrement(); + + // Send valid, yet unexpected message type in response, so peer doesn't have to wait for timeout + LOGGER.debug(() -> String.format("Sending 'account unknown' response to peer %s for GET_ACCOUNT request for unknown account %s", peer, address)); + + // We'll send empty block summaries message as it's very short + Message accountUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + accountUnknownMessage.setId(message.getId()); + if (!peer.sendMessage(accountUnknownMessage)) + peer.disconnect("failed to send account-unknown response"); + return; + } + + AccountMessage accountMessage = new AccountMessage(accountData); + accountMessage.setId(message.getId()); + + if (!peer.sendMessage(accountMessage)) { + peer.disconnect("failed to send account"); + } + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while send account %s to peer %s", address, peer), e); + } + } + + private void onNetworkGetAccountBalanceMessage(Peer peer, Message message) { + GetAccountBalanceMessage getAccountBalanceMessage = (GetAccountBalanceMessage) message; + String address = getAccountBalanceMessage.getAddress(); + long assetId = getAccountBalanceMessage.getAssetId(); + this.stats.getAccountBalanceMessageStats.requests.incrementAndGet(); + + try (final Repository repository = RepositoryManager.getRepository()) { + AccountBalanceData accountBalanceData = repository.getAccountRepository().getBalance(address, assetId); + + if (accountBalanceData == null) { + // We don't have this account + this.stats.getAccountBalanceMessageStats.unknownAccounts.getAndIncrement(); + + // Send valid, yet unexpected message type in response, so peer doesn't have to wait for timeout + LOGGER.debug(() -> String.format("Sending 'account unknown' response to peer %s for GET_ACCOUNT_BALANCE request for unknown account %s and asset ID %d", peer, address, assetId)); + + // We'll send empty block summaries message as it's very short + Message accountUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + accountUnknownMessage.setId(message.getId()); + if (!peer.sendMessage(accountUnknownMessage)) + peer.disconnect("failed to send account-unknown response"); + return; + } + + AccountBalanceMessage accountMessage = new AccountBalanceMessage(accountBalanceData); + accountMessage.setId(message.getId()); + + if (!peer.sendMessage(accountMessage)) { + peer.disconnect("failed to send account balance"); + } + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while send balance for account %s and asset ID %d to peer %s", address, assetId, peer), e); + } + } + + private void onNetworkGetAccountTransactionsMessage(Peer peer, Message message) { + GetAccountTransactionsMessage getAccountTransactionsMessage = (GetAccountTransactionsMessage) message; + String address = getAccountTransactionsMessage.getAddress(); + int limit = Math.min(getAccountTransactionsMessage.getLimit(), 100); + int offset = getAccountTransactionsMessage.getOffset(); + this.stats.getAccountTransactionsMessageStats.requests.incrementAndGet(); + + try (final Repository repository = RepositoryManager.getRepository()) { + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, + null, null, null, address, TransactionsResource.ConfirmationStatus.CONFIRMED, limit, offset, false); + + // Expand signatures to transactions + List transactions = new ArrayList<>(signatures.size()); + for (byte[] signature : signatures) { + transactions.add(repository.getTransactionRepository().fromSignature(signature)); + } + + if (transactions == null) { + // We don't have this account + this.stats.getAccountTransactionsMessageStats.unknownAccounts.getAndIncrement(); + + // Send valid, yet unexpected message type in response, so peer doesn't have to wait for timeout + LOGGER.debug(() -> String.format("Sending 'account unknown' response to peer %s for GET_ACCOUNT_TRANSACTIONS request for unknown account %s", peer, address)); + + // We'll send empty block summaries message as it's very short + Message accountUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + accountUnknownMessage.setId(message.getId()); + if (!peer.sendMessage(accountUnknownMessage)) + peer.disconnect("failed to send account-unknown response"); + return; + } + + TransactionsMessage transactionsMessage = new TransactionsMessage(transactions); + transactionsMessage.setId(message.getId()); + + if (!peer.sendMessage(transactionsMessage)) { + peer.disconnect("failed to send account transactions"); + } + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while sending transactions for account %s %d to peer %s", address, peer), e); + } catch (MessageException e) { + LOGGER.error(String.format("Message serialization issue while sending transactions for account %s %d to peer %s", address, peer), e); + } + } + + private void onNetworkGetAccountNamesMessage(Peer peer, Message message) { + GetAccountNamesMessage getAccountNamesMessage = (GetAccountNamesMessage) message; + String address = getAccountNamesMessage.getAddress(); + this.stats.getAccountNamesMessageStats.requests.incrementAndGet(); + + try (final Repository repository = RepositoryManager.getRepository()) { + List namesDataList = repository.getNameRepository().getNamesByOwner(address); + + if (namesDataList == null) { + // We don't have this account + this.stats.getAccountNamesMessageStats.unknownAccounts.getAndIncrement(); + + // Send valid, yet unexpected message type in response, so peer doesn't have to wait for timeout + LOGGER.debug(() -> String.format("Sending 'account unknown' response to peer %s for GET_ACCOUNT_NAMES request for unknown account %s", peer, address)); + + // We'll send empty block summaries message as it's very short + Message accountUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + accountUnknownMessage.setId(message.getId()); + if (!peer.sendMessage(accountUnknownMessage)) + peer.disconnect("failed to send account-unknown response"); + return; + } + + NamesMessage namesMessage = new NamesMessage(namesDataList); + namesMessage.setId(message.getId()); + + if (!peer.sendMessage(namesMessage)) { + peer.disconnect("failed to send account names"); + } + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while send names for account %s to peer %s", address, peer), e); + } + } + + private void onNetworkGetNameMessage(Peer peer, Message message) { + GetNameMessage getNameMessage = (GetNameMessage) message; + String name = getNameMessage.getName(); + this.stats.getNameMessageStats.requests.incrementAndGet(); + + try (final Repository repository = RepositoryManager.getRepository()) { + NameData nameData = repository.getNameRepository().fromName(name); + + if (nameData == null) { + // We don't have this account + this.stats.getNameMessageStats.unknownAccounts.getAndIncrement(); + + // Send valid, yet unexpected message type in response, so peer doesn't have to wait for timeout + LOGGER.debug(() -> String.format("Sending 'name unknown' response to peer %s for GET_NAME request for unknown name %s", peer, name)); + + // We'll send empty block summaries message as it's very short + Message nameUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + nameUnknownMessage.setId(message.getId()); + if (!peer.sendMessage(nameUnknownMessage)) + peer.disconnect("failed to send name-unknown response"); + return; + } + + NamesMessage namesMessage = new NamesMessage(Arrays.asList(nameData)); + namesMessage.setId(message.getId()); + + if (!peer.sendMessage(namesMessage)) { + peer.disconnect("failed to send name data"); + } + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while send name %s to peer %s", name, peer), e); + } + } + // Utilities @@ -1505,6 +1781,11 @@ public class Controller extends Thread { * @return boolean - whether our node's blockchain is up to date or not */ public boolean isUpToDate(Long minLatestBlockTimestamp) { + if (Settings.getInstance().isLite()) { + // Lite nodes are always "up to date" + return true; + } + // Do we even have a vaguely recent block? if (minLatestBlockTimestamp == null) return false; diff --git a/src/main/java/org/qortal/controller/LiteNode.java b/src/main/java/org/qortal/controller/LiteNode.java new file mode 100644 index 00000000..cfbe8321 --- /dev/null +++ b/src/main/java/org/qortal/controller/LiteNode.java @@ -0,0 +1,189 @@ +package org.qortal.controller; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.data.account.AccountBalanceData; +import org.qortal.data.account.AccountData; +import org.qortal.data.naming.NameData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.*; + +import java.security.SecureRandom; +import java.util.*; + +import static org.qortal.network.message.MessageType.*; + +public class LiteNode { + + private static final Logger LOGGER = LogManager.getLogger(LiteNode.class); + + private static LiteNode instance; + + + public Map pendingRequests = Collections.synchronizedMap(new HashMap<>()); + + public int MAX_TRANSACTIONS_PER_MESSAGE = 100; + + + public LiteNode() { + + } + + public static synchronized LiteNode getInstance() { + if (instance == null) { + instance = new LiteNode(); + } + + return instance; + } + + + /** + * Fetch account data from peers for given QORT address + * @param address - the QORT address to query + * @return accountData - the account data for this address, or null if not retrieved + */ + public AccountData fetchAccountData(String address) { + GetAccountMessage getAccountMessage = new GetAccountMessage(address); + AccountMessage accountMessage = (AccountMessage) this.sendMessage(getAccountMessage, ACCOUNT); + if (accountMessage == null) { + return null; + } + return accountMessage.getAccountData(); + } + + /** + * Fetch account balance data from peers for given QORT address and asset ID + * @param address - the QORT address to query + * @return balance - the balance for this address and assetId, or null if not retrieved + */ + public AccountBalanceData fetchAccountBalance(String address, long assetId) { + GetAccountBalanceMessage getAccountMessage = new GetAccountBalanceMessage(address, assetId); + AccountBalanceMessage accountMessage = (AccountBalanceMessage) this.sendMessage(getAccountMessage, ACCOUNT_BALANCE); + if (accountMessage == null) { + return null; + } + return accountMessage.getAccountBalanceData(); + } + + /** + * Fetch list of transactions for given QORT address + * @param address - the QORT address to query + * @param limit - the maximum number of results to return + * @param offset - the starting index + * @return a list of TransactionData objects, or null if not retrieved + */ + public List fetchAccountTransactions(String address, int limit, int offset) { + List allTransactions = new ArrayList<>(); + if (limit == 0) { + limit = Integer.MAX_VALUE; + } + int batchSize = Math.min(limit, MAX_TRANSACTIONS_PER_MESSAGE); + + while (allTransactions.size() < limit) { + GetAccountTransactionsMessage getAccountTransactionsMessage = new GetAccountTransactionsMessage(address, batchSize, offset); + TransactionsMessage transactionsMessage = (TransactionsMessage) this.sendMessage(getAccountTransactionsMessage, TRANSACTIONS); + if (transactionsMessage == null) { + // An error occurred, so give up instead of returning partial results + return null; + } + allTransactions.addAll(transactionsMessage.getTransactions()); + if (transactionsMessage.getTransactions().size() < batchSize) { + // No more transactions to fetch + break; + } + offset += batchSize; + } + return allTransactions; + } + + /** + * Fetch list of names for given QORT address + * @param address - the QORT address to query + * @return a list of NameData objects, or null if not retrieved + */ + public List fetchAccountNames(String address) { + GetAccountNamesMessage getAccountNamesMessage = new GetAccountNamesMessage(address); + NamesMessage namesMessage = (NamesMessage) this.sendMessage(getAccountNamesMessage, NAMES); + if (namesMessage == null) { + return null; + } + return namesMessage.getNameDataList(); + } + + /** + * Fetch info about a registered name + * @param name - the name to query + * @return a NameData object, or null if not retrieved + */ + public NameData fetchNameData(String name) { + GetNameMessage getNameMessage = new GetNameMessage(name); + NamesMessage namesMessage = (NamesMessage) this.sendMessage(getNameMessage, NAMES); + if (namesMessage == null) { + return null; + } + List nameDataList = namesMessage.getNameDataList(); + if (nameDataList == null || nameDataList.size() != 1) { + return null; + } + // We are only expecting a single item in the list + return nameDataList.get(0); + } + + + private Message sendMessage(Message message, MessageType expectedResponseMessageType) { + // This asks a random peer for the data + // TODO: ask multiple peers, and disregard everything if there are any significant differences in the responses + + // Needs a mutable copy of the unmodifiableList + List peers = new ArrayList<>(Network.getInstance().getImmutableHandshakedPeers()); + + // Disregard peers that have "misbehaved" recently + peers.removeIf(Controller.hasMisbehaved); + + // Disregard peers that only have genesis block + // TODO: peers.removeIf(Controller.hasOnlyGenesisBlock); + + // Disregard peers that are on an old version + peers.removeIf(Controller.hasOldVersion); + + // Disregard peers that are on a known inferior chain tip + // TODO: peers.removeIf(Controller.hasInferiorChainTip); + + if (peers.isEmpty()) { + LOGGER.info("No peers available to send {} message to", message.getType()); + return null; + } + + // Pick random peer + int index = new SecureRandom().nextInt(peers.size()); + Peer peer = peers.get(index); + + LOGGER.info("Sending {} message to peer {}...", message.getType(), peer); + + Message responseMessage; + + try { + responseMessage = peer.getResponse(message); + + } catch (InterruptedException e) { + return null; + } + + if (responseMessage == null) { + LOGGER.info("Peer didn't respond to {} message", peer, message.getType()); + return null; + } + else if (responseMessage.getType() != expectedResponseMessageType) { + LOGGER.info("Peer responded with unexpected message type {} (should be {})", peer, responseMessage.getType(), expectedResponseMessageType); + return null; + } + + LOGGER.info("Peer {} responded with {} message", peer, responseMessage.getType()); + + return responseMessage; + } + +} diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 55aeae04..8f3a34bb 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -134,6 +134,11 @@ public class Synchronizer extends Thread { public void run() { Thread.currentThread().setName("Synchronizer"); + if (Settings.getInstance().isLite()) { + // Lite nodes don't need to sync + return; + } + try { while (running && !Controller.isStopping()) { Thread.sleep(1000); diff --git a/src/main/java/org/qortal/controller/TransactionImporter.java b/src/main/java/org/qortal/controller/TransactionImporter.java index 16fd3a59..39f45a14 100644 --- a/src/main/java/org/qortal/controller/TransactionImporter.java +++ b/src/main/java/org/qortal/controller/TransactionImporter.java @@ -11,6 +11,7 @@ import org.qortal.network.message.TransactionSignaturesMessage; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; import org.qortal.transaction.Transaction; import org.qortal.transform.TransformationException; import org.qortal.utils.Base58; @@ -19,6 +20,7 @@ import org.qortal.utils.NTP; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; public class TransactionImporter extends Thread { @@ -55,12 +57,16 @@ public class TransactionImporter extends Thread { @Override public void run() { + Thread.currentThread().setName("Transaction Importer"); + try { while (!Controller.isStopping()) { Thread.sleep(1000L); // Process incoming transactions queue - processIncomingTransactionsQueue(); + validateTransactionsInQueue(); + importTransactionsInQueue(); + // Clean up invalid incoming transactions list cleanupInvalidTransactionsList(NTP.getTime()); } @@ -87,7 +93,24 @@ public class TransactionImporter extends Thread { incomingTransactions.keySet().removeIf(t -> Arrays.equals(t.getSignature(), signature)); } - private void processIncomingTransactionsQueue() { + /** + * Retrieve all pending unconfirmed transactions that have had their signatures validated. + * @return a list of TransactionData objects, with valid signatures. + */ + private List getCachedSigValidTransactions() { + return this.incomingTransactions.entrySet().stream() + .filter(t -> Boolean.TRUE.equals(t.getValue())) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } + + /** + * Validate the signatures of any transactions pending import, then update their + * entries in the queue to mark them as valid/invalid. + * + * No database lock is required. + */ + private void validateTransactionsInQueue() { if (this.incomingTransactions.isEmpty()) { // Nothing to do? return; @@ -106,6 +129,8 @@ public class TransactionImporter extends Thread { List sigValidTransactions = new ArrayList<>(); + boolean isLiteNode = Settings.getInstance().isLite(); + // Signature validation round - does not require blockchain lock for (Map.Entry transactionEntry : incomingTransactionsCopy.entrySet()) { // Quick exit? @@ -119,17 +144,25 @@ public class TransactionImporter extends Thread { // Only validate signature if we haven't already done so Boolean isSigValid = transactionEntry.getValue(); if (!Boolean.TRUE.equals(isSigValid)) { + if (isLiteNode) { + // Lite nodes can't easily validate transactions, so for now we will have to assume that everything is valid + sigValidTransactions.add(transaction); + // Add mark signature as valid if transaction still exists in import queue + incomingTransactions.computeIfPresent(transactionData, (k, v) -> Boolean.TRUE); + continue; + } + if (!transaction.isSignatureValid()) { String signature58 = Base58.encode(transactionData.getSignature()); - LOGGER.trace("Ignoring {} transaction {} with invalid signature", transactionData.getType().name(), signature58); + LOGGER.debug("Ignoring {} transaction {} with invalid signature", transactionData.getType().name(), signature58); removeIncomingTransaction(transactionData.getSignature()); // Also add to invalidIncomingTransactions map Long now = NTP.getTime(); if (now != null) { Long expiry = now + INVALID_TRANSACTION_RECHECK_INTERVAL; - LOGGER.trace("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58); + LOGGER.trace("Adding invalid transaction {} to invalidUnconfirmedTransactions...", signature58); // Add to invalidUnconfirmedTransactions so that we don't keep requesting it invalidUnconfirmedTransactions.put(signature58, expiry); } @@ -155,30 +188,43 @@ public class TransactionImporter extends Thread { LOGGER.debug("Finished validating signatures in incoming transactions queue (valid this round: {}, total pending import: {})...", validatedCount, sigValidTransactions.size()); } - if (sigValidTransactions.isEmpty()) { - // Don't bother locking if there are no new transactions to process + } catch (DataException e) { + LOGGER.error("Repository issue while processing incoming transactions", e); + } + } + + /** + * Import any transactions in the queue that have valid signatures. + * + * A database lock is required. + */ + private void importTransactionsInQueue() { + List sigValidTransactions = this.getCachedSigValidTransactions(); + if (sigValidTransactions.isEmpty()) { + // Don't bother locking if there are no new transactions to process + return; + } + + if (Synchronizer.getInstance().isSyncRequested() || Synchronizer.getInstance().isSynchronizing()) { + // Prioritize syncing, and don't attempt to lock + return; + } + + try { + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + if (!blockchainLock.tryLock(2, TimeUnit.SECONDS)) { + LOGGER.debug("Too busy to import incoming transactions queue"); return; } + } catch (InterruptedException e) { + LOGGER.debug("Interrupted when trying to acquire blockchain lock"); + return; + } - if (Synchronizer.getInstance().isSyncRequested() || Synchronizer.getInstance().isSynchronizing()) { - // Prioritize syncing, and don't attempt to lock - // Signature validity is retained in the incomingTransactions map, to avoid the above work being wasted - return; - } + LOGGER.debug("Importing incoming transactions queue (size {})...", sigValidTransactions.size()); - try { - ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - if (!blockchainLock.tryLock(2, TimeUnit.SECONDS)) { - // Signature validity is retained in the incomingTransactions map, to avoid the above work being wasted - LOGGER.debug("Too busy to process incoming transactions queue"); - return; - } - } catch (InterruptedException e) { - LOGGER.debug("Interrupted when trying to acquire blockchain lock"); - return; - } - - LOGGER.debug("Processing incoming transactions queue (size {})...", sigValidTransactions.size()); + int processedCount = 0; + try (final Repository repository = RepositoryManager.getRepository()) { // Import transactions with valid signatures try { @@ -188,14 +234,15 @@ public class TransactionImporter extends Thread { } if (Synchronizer.getInstance().isSyncRequestPending()) { - LOGGER.debug("Breaking out of transaction processing with {} remaining, because a sync request is pending", sigValidTransactions.size() - i); + LOGGER.debug("Breaking out of transaction importing with {} remaining, because a sync request is pending", sigValidTransactions.size() - i); return; } - Transaction transaction = sigValidTransactions.get(i); - TransactionData transactionData = transaction.getTransactionData(); + TransactionData transactionData = sigValidTransactions.get(i); + Transaction transaction = Transaction.fromData(repository, transactionData); Transaction.ValidationResult validationResult = transaction.importAsUnconfirmed(); + processedCount++; switch (validationResult) { case TRANSACTION_ALREADY_EXISTS: { @@ -217,7 +264,7 @@ public class TransactionImporter extends Thread { // All other invalid cases: default: { final String signature58 = Base58.encode(transactionData.getSignature()); - LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58)); + LOGGER.debug(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58)); Long now = NTP.getTime(); if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) { @@ -240,12 +287,12 @@ public class TransactionImporter extends Thread { removeIncomingTransaction(transactionData.getSignature()); } } finally { - LOGGER.debug("Finished processing incoming transactions queue"); + LOGGER.debug("Finished importing {} incoming transaction{}", processedCount, (processedCount == 1 ? "" : "s")); ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); blockchainLock.unlock(); } } catch (DataException e) { - LOGGER.error("Repository issue while processing incoming transactions", e); + LOGGER.error("Repository issue while importing incoming transactions", e); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index 05a45425..a0b4886b 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -283,8 +283,8 @@ public class ArbitraryDataFileListManager { LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size())); - // FUTURE: send our address as requestingPeer once enough peers have switched to the new protocol - String requestingPeer = null; // Network.getInstance().getOurExternalIpAddressAndPort(); + // Send our address as requestingPeer, to allow for potential direct connections with seeds/peers + String requestingPeer = Network.getInstance().getOurExternalIpAddressAndPort(); // Build request Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, missingHashes, now, 0, requestingPeer); @@ -636,6 +636,9 @@ public class ArbitraryDataFileListManager { // We should only respond if we have at least one hash if (hashes.size() > 0) { + // Firstly we should keep track of the requesting peer, to allow for potential direct connections later + ArbitraryDataFileManager.getInstance().addRecentDataRequest(requestingPeer); + // We have all the chunks, so update requests map to reflect that we've sent it // There is no need to keep track of the request, as we can serve all the chunks if (allChunksExist) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 11e15414..22cf4144 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -1,5 +1,6 @@ package org.qortal.controller.arbitrary; +import com.google.common.net.InetAddresses; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataFile; @@ -54,6 +55,13 @@ public class ArbitraryDataFileManager extends Thread { */ private List directConnectionInfo = Collections.synchronizedList(new ArrayList<>()); + /** + * Map to keep track of peers requesting QDN data that we hold. + * Key = peer address string, value = time of last request. + * This allows for additional "burst" connections beyond existing limits. + */ + private Map recentDataRequests = Collections.synchronizedMap(new HashMap<>()); + public static int MAX_FILE_HASH_RESPONSES = 1000; @@ -108,6 +116,9 @@ public class ArbitraryDataFileManager extends Thread { final long directConnectionInfoMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_DIRECT_CONNECTION_INFO_TIMEOUT; directConnectionInfo.removeIf(entry -> entry.getTimestamp() < directConnectionInfoMinimumTimestamp); + + final long recentDataRequestMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RECENT_DATA_REQUESTS_TIMEOUT; + recentDataRequests.entrySet().removeIf(entry -> entry.getValue() < recentDataRequestMinimumTimestamp); } @@ -490,6 +501,45 @@ public class ArbitraryDataFileManager extends Thread { } + // Peers requesting QDN data from us + + /** + * Add an address string of a peer that is trying to request data from us. + * @param peerAddress + */ + public void addRecentDataRequest(String peerAddress) { + if (peerAddress == null) { + return; + } + + Long now = NTP.getTime(); + if (now == null) { + return; + } + + // Make sure to remove the port, since it isn't guaranteed to match next time + String[] parts = peerAddress.split(":"); + if (parts.length == 0) { + return; + } + String host = parts[0]; + if (!InetAddresses.isInetAddress(host)) { + // Invalid host + return; + } + + this.recentDataRequests.put(host, now); + } + + public boolean isPeerRequestingData(String peerAddressWithoutPort) { + return this.recentDataRequests.containsKey(peerAddressWithoutPort); + } + + public boolean hasPendingDataRequest() { + return !this.recentDataRequests.isEmpty(); + } + + // Network handlers public void onNetworkGetArbitraryDataFileMessage(Peer peer, Message message) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 4b6d3a28..6b3f0160 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -47,6 +47,9 @@ public class ArbitraryDataManager extends Thread { /** Maximum time to hold direct peer connection information */ public static final long ARBITRARY_DIRECT_CONNECTION_INFO_TIMEOUT = 2 * 60 * 1000L; // ms + /** Maximum time to hold information about recent data requests that we can fulfil */ + public static final long ARBITRARY_RECENT_DATA_REQUESTS_TIMEOUT = 2 * 60 * 1000L; // ms + /** Maximum number of hops that an arbitrary signatures request is allowed to make */ private static int ARBITRARY_SIGNATURES_REQUEST_MAX_HOPS = 3; diff --git a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java index 54fba699..bd12f784 100644 --- a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java @@ -19,6 +19,11 @@ public class AtStatesPruner implements Runnable { public void run() { Thread.currentThread().setName("AT States pruner"); + if (Settings.getInstance().isLite()) { + // Nothing to prune in lite mode + return; + } + boolean archiveMode = false; if (!Settings.getInstance().isTopOnly()) { // Top-only mode isn't enabled, but we might want to prune for the purposes of archiving diff --git a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java index d3bdc345..69fa347c 100644 --- a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java @@ -19,6 +19,11 @@ public class AtStatesTrimmer implements Runnable { public void run() { Thread.currentThread().setName("AT States trimmer"); + if (Settings.getInstance().isLite()) { + // Nothing to trim in lite mode + return; + } + try (final Repository repository = RepositoryManager.getRepository()) { int trimStartHeight = repository.getATRepository().getAtTrimHeight(); diff --git a/src/main/java/org/qortal/controller/repository/BlockArchiver.java b/src/main/java/org/qortal/controller/repository/BlockArchiver.java index ef26610c..8757bf32 100644 --- a/src/main/java/org/qortal/controller/repository/BlockArchiver.java +++ b/src/main/java/org/qortal/controller/repository/BlockArchiver.java @@ -21,7 +21,7 @@ public class BlockArchiver implements Runnable { public void run() { Thread.currentThread().setName("Block archiver"); - if (!Settings.getInstance().isArchiveEnabled()) { + if (!Settings.getInstance().isArchiveEnabled() || Settings.getInstance().isLite()) { return; } diff --git a/src/main/java/org/qortal/controller/repository/BlockPruner.java b/src/main/java/org/qortal/controller/repository/BlockPruner.java index 03fb38b9..23e3a45a 100644 --- a/src/main/java/org/qortal/controller/repository/BlockPruner.java +++ b/src/main/java/org/qortal/controller/repository/BlockPruner.java @@ -19,6 +19,11 @@ public class BlockPruner implements Runnable { public void run() { Thread.currentThread().setName("Block pruner"); + if (Settings.getInstance().isLite()) { + // Nothing to prune in lite mode + return; + } + boolean archiveMode = false; if (!Settings.getInstance().isTopOnly()) { // Top-only mode isn't enabled, but we might want to prune for the purposes of archiving diff --git a/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java b/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java index dfd9d45e..d74df4b5 100644 --- a/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java +++ b/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java @@ -21,6 +21,11 @@ public class OnlineAccountsSignaturesTrimmer implements Runnable { public void run() { Thread.currentThread().setName("Online Accounts trimmer"); + if (Settings.getInstance().isLite()) { + // Nothing to trim in lite mode + return; + } + try (final Repository repository = RepositoryManager.getRepository()) { // Don't even start trimming until initial rush has ended Thread.sleep(INITIAL_SLEEP_PERIOD); diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index a04509f1..6bc58bb4 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -8,6 +8,7 @@ import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters; import org.qortal.block.BlockChain; import org.qortal.controller.Controller; import org.qortal.controller.arbitrary.ArbitraryDataFileListManager; +import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.crypto.Crypto; import org.qortal.data.block.BlockData; import org.qortal.data.network.PeerData; @@ -259,6 +260,18 @@ public class Network { return this.immutableConnectedPeers; } + public List getImmutableConnectedDataPeers() { + return this.getImmutableConnectedPeers().stream() + .filter(p -> p.isDataPeer()) + .collect(Collectors.toList()); + } + + public List getImmutableConnectedNonDataPeers() { + return this.getImmutableConnectedPeers().stream() + .filter(p -> !p.isDataPeer()) + .collect(Collectors.toList()); + } + public void addConnectedPeer(Peer peer) { this.connectedPeers.add(peer); // thread safe thanks to synchronized list this.immutableConnectedPeers = List.copyOf(this.connectedPeers); // also thread safe thanks to synchronized collection's toArray() being fed to List.of(array) @@ -325,6 +338,7 @@ public class Network { // Add this signature to the list of pending requests for this peer LOGGER.info("Making connection to peer {} to request files for signature {}...", peerAddressString, Base58.encode(signature)); Peer peer = new Peer(peerData); + peer.setIsDataPeer(true); peer.addPendingSignatureRequest(signature); return this.connectPeer(peer); // If connection (and handshake) is successful, data will automatically be requested @@ -685,6 +699,7 @@ public class Network { // Pick candidate PeerData peerData = peers.get(peerIndex); Peer newPeer = new Peer(peerData); + newPeer.setIsDataPeer(false); // Update connection attempt info peerData.setLastAttempted(now); @@ -1069,11 +1084,13 @@ public class Network { // (If inbound sent anything here, it's possible it could be processed out-of-order with handshake message). if (peer.isOutbound()) { - // Send our height - Message heightMessage = buildHeightMessage(peer, Controller.getInstance().getChainTip()); - if (!peer.sendMessage(heightMessage)) { - peer.disconnect("failed to send height/info"); - return; + if (!Settings.getInstance().isLite()) { + // Send our height + Message heightMessage = buildHeightMessage(peer, Controller.getInstance().getChainTip()); + if (!peer.sendMessage(heightMessage)) { + peer.disconnect("failed to send height/info"); + return; + } } // Send our peers list diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index dbb03fda..7e51dc36 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -64,6 +64,11 @@ public class Peer { */ private boolean isLocal; + /** + * True if connected for the purposes of transfering specific QDN data + */ + private boolean isDataPeer; + private final UUID peerConnectionId = UUID.randomUUID(); private final Object byteBufferLock = new Object(); private ByteBuffer byteBuffer; @@ -194,6 +199,14 @@ public class Peer { return this.isOutbound; } + public boolean isDataPeer() { + return isDataPeer; + } + + public void setIsDataPeer(boolean isDataPeer) { + this.isDataPeer = isDataPeer; + } + public Handshake getHandshakeStatus() { synchronized (this.handshakingLock) { return this.handshakeStatus; @@ -211,6 +224,11 @@ public class Peer { } private void generateRandomMaxConnectionAge() { + if (this.maxConnectionAge > 0L) { + // Already generated, so we don't want to overwrite the existing value + return; + } + // Retrieve the min and max connection time from the settings, and calculate the range final int minPeerConnectionTime = Settings.getInstance().getMinPeerConnectionTime(); final int maxPeerConnectionTime = Settings.getInstance().getMaxPeerConnectionTime(); @@ -893,6 +911,10 @@ public class Peer { return maxConnectionAge; } + public void setMaxConnectionAge(long maxConnectionAge) { + this.maxConnectionAge = maxConnectionAge; + } + public boolean hasReachedMaxConnectionAge() { return this.getConnectionAge() > this.getMaxConnectionAge(); } diff --git a/src/main/java/org/qortal/network/message/AccountBalanceMessage.java b/src/main/java/org/qortal/network/message/AccountBalanceMessage.java new file mode 100644 index 00000000..7a9ad725 --- /dev/null +++ b/src/main/java/org/qortal/network/message/AccountBalanceMessage.java @@ -0,0 +1,70 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Longs; +import org.qortal.data.account.AccountBalanceData; +import org.qortal.transform.Transformer; +import org.qortal.utils.Base58; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class AccountBalanceMessage extends Message { + + private static final int ADDRESS_LENGTH = Transformer.ADDRESS_LENGTH; + + private AccountBalanceData accountBalanceData; + + public AccountBalanceMessage(AccountBalanceData accountBalanceData) { + super(MessageType.ACCOUNT_BALANCE); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + // Send raw address instead of base58 encoded + byte[] address = Base58.decode(accountBalanceData.getAddress()); + bytes.write(address); + + bytes.write(Longs.toByteArray(accountBalanceData.getAssetId())); + + bytes.write(Longs.toByteArray(accountBalanceData.getBalance())); + + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + public AccountBalanceMessage(int id, AccountBalanceData accountBalanceData) { + super(id, MessageType.ACCOUNT_BALANCE); + + this.accountBalanceData = accountBalanceData; + } + + public AccountBalanceData getAccountBalanceData() { + return this.accountBalanceData; + } + + + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) { + byte[] addressBytes = new byte[ADDRESS_LENGTH]; + byteBuffer.get(addressBytes); + String address = Base58.encode(addressBytes); + + long assetId = byteBuffer.getLong(); + + long balance = byteBuffer.getLong(); + + AccountBalanceData accountBalanceData = new AccountBalanceData(address, assetId, balance); + return new AccountBalanceMessage(id, accountBalanceData); + } + + public AccountBalanceMessage cloneWithNewId(int newId) { + AccountBalanceMessage clone = new AccountBalanceMessage(this.accountBalanceData); + clone.setId(newId); + return clone; + } + +} diff --git a/src/main/java/org/qortal/network/message/AccountMessage.java b/src/main/java/org/qortal/network/message/AccountMessage.java new file mode 100644 index 00000000..d22ef879 --- /dev/null +++ b/src/main/java/org/qortal/network/message/AccountMessage.java @@ -0,0 +1,93 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import org.qortal.data.account.AccountData; +import org.qortal.transform.Transformer; +import org.qortal.utils.Base58; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class AccountMessage extends Message { + + private static final int ADDRESS_LENGTH = Transformer.ADDRESS_LENGTH; + private static final int REFERENCE_LENGTH = Transformer.SIGNATURE_LENGTH; + private static final int PUBLIC_KEY_LENGTH = Transformer.PUBLIC_KEY_LENGTH; + + private AccountData accountData; + + public AccountMessage(AccountData accountData) { + super(MessageType.ACCOUNT); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + // Send raw address instead of base58 encoded + byte[] address = Base58.decode(accountData.getAddress()); + bytes.write(address); + + bytes.write(accountData.getReference()); + + bytes.write(accountData.getPublicKey()); + + bytes.write(Ints.toByteArray(accountData.getDefaultGroupId())); + + bytes.write(Ints.toByteArray(accountData.getFlags())); + + bytes.write(Ints.toByteArray(accountData.getLevel())); + + bytes.write(Ints.toByteArray(accountData.getBlocksMinted())); + + bytes.write(Ints.toByteArray(accountData.getBlocksMintedAdjustment())); + + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + public AccountMessage(int id, AccountData accountData) { + super(id, MessageType.ACCOUNT); + + this.accountData = accountData; + } + + public AccountData getAccountData() { + return this.accountData; + } + + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) { + byte[] addressBytes = new byte[ADDRESS_LENGTH]; + byteBuffer.get(addressBytes); + String address = Base58.encode(addressBytes); + + byte[] reference = new byte[REFERENCE_LENGTH]; + byteBuffer.get(reference); + + byte[] publicKey = new byte[PUBLIC_KEY_LENGTH]; + byteBuffer.get(publicKey); + + int defaultGroupId = byteBuffer.getInt(); + + int flags = byteBuffer.getInt(); + + int level = byteBuffer.getInt(); + + int blocksMinted = byteBuffer.getInt(); + + int blocksMintedAdjustment = byteBuffer.getInt(); + + AccountData accountData = new AccountData(address, reference, publicKey, defaultGroupId, flags, level, blocksMinted, blocksMintedAdjustment); + return new AccountMessage(id, accountData); + } + + public AccountMessage cloneWithNewId(int newId) { + AccountMessage clone = new AccountMessage(this.accountData); + clone.setId(newId); + return clone; + } + +} diff --git a/src/main/java/org/qortal/network/message/GetAccountBalanceMessage.java b/src/main/java/org/qortal/network/message/GetAccountBalanceMessage.java new file mode 100644 index 00000000..43892b83 --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetAccountBalanceMessage.java @@ -0,0 +1,63 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Longs; +import org.qortal.transform.Transformer; +import org.qortal.utils.Base58; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class GetAccountBalanceMessage extends Message { + + private static final int ADDRESS_LENGTH = Transformer.ADDRESS_LENGTH; + + private String address; + private long assetId; + + public GetAccountBalanceMessage(String address, long assetId) { + super(MessageType.GET_ACCOUNT_BALANCE); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + // Send raw address instead of base58 encoded + byte[] addressBytes = Base58.decode(address); + bytes.write(addressBytes); + + bytes.write(Longs.toByteArray(assetId)); + + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private GetAccountBalanceMessage(int id, String address, long assetId) { + super(id, MessageType.GET_ACCOUNT_BALANCE); + + this.address = address; + this.assetId = assetId; + } + + public String getAddress() { + return this.address; + } + + public long getAssetId() { + return this.assetId; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] addressBytes = new byte[ADDRESS_LENGTH]; + bytes.get(addressBytes); + String address = Base58.encode(addressBytes); + + long assetId = bytes.getLong(); + + return new GetAccountBalanceMessage(id, address, assetId); + } + +} diff --git a/src/main/java/org/qortal/network/message/GetAccountMessage.java b/src/main/java/org/qortal/network/message/GetAccountMessage.java new file mode 100644 index 00000000..4f2a6dec --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetAccountMessage.java @@ -0,0 +1,56 @@ +package org.qortal.network.message; + +import org.qortal.transform.Transformer; +import org.qortal.utils.Base58; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; + +public class GetAccountMessage extends Message { + + private static final int ADDRESS_LENGTH = Transformer.ADDRESS_LENGTH; + + private String address; + + public GetAccountMessage(String address) { + super(MessageType.GET_ACCOUNT); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + // Send raw address instead of base58 encoded + byte[] addressBytes = Base58.decode(address); + bytes.write(addressBytes); + + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private GetAccountMessage(int id, String address) { + super(id, MessageType.GET_ACCOUNT); + + this.address = address; + } + + public String getAddress() { + return this.address; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + if (bytes.remaining() != ADDRESS_LENGTH) + throw new BufferUnderflowException(); + + byte[] addressBytes = new byte[ADDRESS_LENGTH]; + bytes.get(addressBytes); + String address = Base58.encode(addressBytes); + + return new GetAccountMessage(id, address); + } + +} diff --git a/src/main/java/org/qortal/network/message/GetAccountNamesMessage.java b/src/main/java/org/qortal/network/message/GetAccountNamesMessage.java new file mode 100644 index 00000000..bde697c5 --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetAccountNamesMessage.java @@ -0,0 +1,53 @@ +package org.qortal.network.message; + +import org.qortal.transform.Transformer; +import org.qortal.utils.Base58; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class GetAccountNamesMessage extends Message { + + private static final int ADDRESS_LENGTH = Transformer.ADDRESS_LENGTH; + + private String address; + + public GetAccountNamesMessage(String address) { + super(MessageType.GET_ACCOUNT_NAMES); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + // Send raw address instead of base58 encoded + byte[] addressBytes = Base58.decode(address); + bytes.write(addressBytes); + + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private GetAccountNamesMessage(int id, String address) { + super(id, MessageType.GET_ACCOUNT_NAMES); + + this.address = address; + } + + public String getAddress() { + return this.address; + } + + + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] addressBytes = new byte[ADDRESS_LENGTH]; + bytes.get(addressBytes); + String address = Base58.encode(addressBytes); + + return new GetAccountNamesMessage(id, address); + } + +} diff --git a/src/main/java/org/qortal/network/message/GetAccountTransactionsMessage.java b/src/main/java/org/qortal/network/message/GetAccountTransactionsMessage.java new file mode 100644 index 00000000..fe921cc9 --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetAccountTransactionsMessage.java @@ -0,0 +1,69 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import org.qortal.transform.Transformer; +import org.qortal.utils.Base58; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class GetAccountTransactionsMessage extends Message { + + private static final int ADDRESS_LENGTH = Transformer.ADDRESS_LENGTH; + + private String address; + private int limit; + private int offset; + + public GetAccountTransactionsMessage(String address, int limit, int offset) { + super(MessageType.GET_ACCOUNT_TRANSACTIONS); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + // Send raw address instead of base58 encoded + byte[] addressBytes = Base58.decode(address); + bytes.write(addressBytes); + + bytes.write(Ints.toByteArray(limit)); + + bytes.write(Ints.toByteArray(offset)); + + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private GetAccountTransactionsMessage(int id, String address, int limit, int offset) { + super(id, MessageType.GET_ACCOUNT_TRANSACTIONS); + + this.address = address; + this.limit = limit; + this.offset = offset; + } + + public String getAddress() { + return this.address; + } + + public int getLimit() { return this.limit; } + + public int getOffset() { return this.offset; } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] addressBytes = new byte[ADDRESS_LENGTH]; + bytes.get(addressBytes); + String address = Base58.encode(addressBytes); + + int limit = bytes.getInt(); + + int offset = bytes.getInt(); + + return new GetAccountTransactionsMessage(id, address, limit, offset); + } + +} diff --git a/src/main/java/org/qortal/network/message/GetNameMessage.java b/src/main/java/org/qortal/network/message/GetNameMessage.java new file mode 100644 index 00000000..10fae08a --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetNameMessage.java @@ -0,0 +1,53 @@ +package org.qortal.network.message; + +import org.qortal.naming.Name; +import org.qortal.transform.TransformationException; +import org.qortal.utils.Serialization; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class GetNameMessage extends Message { + + private String name; + + public GetNameMessage(String address) { + super(MessageType.GET_NAME); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + Serialization.serializeSizedStringV2(bytes, name); + + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private GetNameMessage(int id, String name) { + super(id, MessageType.GET_NAME); + + this.name = name; + } + + public String getName() { + return this.name; + } + + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { + try { + String name = Serialization.deserializeSizedStringV2(bytes, Name.MAX_NAME_SIZE); + + return new GetNameMessage(id, name); + + } catch (TransformationException e) { + throw new MessageException(e.getMessage(), e); + } + } + +} diff --git a/src/main/java/org/qortal/network/message/MessageType.java b/src/main/java/org/qortal/network/message/MessageType.java index 48039a4d..8ad7a0da 100644 --- a/src/main/java/org/qortal/network/message/MessageType.java +++ b/src/main/java/org/qortal/network/message/MessageType.java @@ -61,7 +61,21 @@ public enum MessageType { GET_TRADE_PRESENCES(141, GetTradePresencesMessage::fromByteBuffer), ARBITRARY_METADATA(150, ArbitraryMetadataMessage::fromByteBuffer), - GET_ARBITRARY_METADATA(151, GetArbitraryMetadataMessage::fromByteBuffer); + GET_ARBITRARY_METADATA(151, GetArbitraryMetadataMessage::fromByteBuffer), + + // Lite node support + ACCOUNT(160, AccountMessage::fromByteBuffer), + GET_ACCOUNT(161, GetAccountMessage::fromByteBuffer), + + ACCOUNT_BALANCE(170, AccountBalanceMessage::fromByteBuffer), + GET_ACCOUNT_BALANCE(171, GetAccountBalanceMessage::fromByteBuffer), + + NAMES(180, NamesMessage::fromByteBuffer), + GET_ACCOUNT_NAMES(181, GetAccountNamesMessage::fromByteBuffer), + GET_NAME(182, GetNameMessage::fromByteBuffer), + + TRANSACTIONS(190, TransactionsMessage::fromByteBuffer), + GET_ACCOUNT_TRANSACTIONS(191, GetAccountTransactionsMessage::fromByteBuffer); public final int value; public final MessageProducer fromByteBufferMethod; diff --git a/src/main/java/org/qortal/network/message/NamesMessage.java b/src/main/java/org/qortal/network/message/NamesMessage.java new file mode 100644 index 00000000..942818cc --- /dev/null +++ b/src/main/java/org/qortal/network/message/NamesMessage.java @@ -0,0 +1,142 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.qortal.data.naming.NameData; +import org.qortal.naming.Name; +import org.qortal.transform.TransformationException; +import org.qortal.transform.Transformer; +import org.qortal.utils.Serialization; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class NamesMessage extends Message { + + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + + private List nameDataList; + + public NamesMessage(List nameDataList) { + super(MessageType.NAMES); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(nameDataList.size())); + + for (int i = 0; i < nameDataList.size(); ++i) { + NameData nameData = nameDataList.get(i); + + Serialization.serializeSizedStringV2(bytes, nameData.getName()); + + Serialization.serializeSizedStringV2(bytes, nameData.getReducedName()); + + Serialization.serializeAddress(bytes, nameData.getOwner()); + + Serialization.serializeSizedStringV2(bytes, nameData.getData()); + + bytes.write(Longs.toByteArray(nameData.getRegistered())); + + Long updated = nameData.getUpdated(); + int wasUpdated = (updated != null) ? 1 : 0; + bytes.write(Ints.toByteArray(wasUpdated)); + + if (updated != null) { + bytes.write(Longs.toByteArray(nameData.getUpdated())); + } + + int isForSale = nameData.isForSale() ? 1 : 0; + bytes.write(Ints.toByteArray(isForSale)); + + if (nameData.isForSale()) { + bytes.write(Longs.toByteArray(nameData.getSalePrice())); + } + + bytes.write(nameData.getReference()); + + bytes.write(Ints.toByteArray(nameData.getCreationGroupId())); + } + + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + public NamesMessage(int id, List nameDataList) { + super(id, MessageType.NAMES); + + this.nameDataList = nameDataList; + } + + public List getNameDataList() { + return this.nameDataList; + } + + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { + try { + final int nameCount = bytes.getInt(); + + List nameDataList = new ArrayList<>(nameCount); + + for (int i = 0; i < nameCount; ++i) { + String name = Serialization.deserializeSizedStringV2(bytes, Name.MAX_NAME_SIZE); + + String reducedName = Serialization.deserializeSizedStringV2(bytes, Name.MAX_NAME_SIZE); + + String owner = Serialization.deserializeAddress(bytes); + + String data = Serialization.deserializeSizedStringV2(bytes, Name.MAX_DATA_SIZE); + + long registered = bytes.getLong(); + + int wasUpdated = bytes.getInt(); + + Long updated = null; + if (wasUpdated == 1) { + updated = bytes.getLong(); + } + + boolean isForSale = (bytes.getInt() == 1); + + Long salePrice = null; + if (isForSale) { + salePrice = bytes.getLong(); + } + + byte[] reference = new byte[SIGNATURE_LENGTH]; + bytes.get(reference); + + int creationGroupId = bytes.getInt(); + + NameData nameData = new NameData(name, reducedName, owner, data, registered, updated, + isForSale, salePrice, reference, creationGroupId); + nameDataList.add(nameData); + } + + if (bytes.hasRemaining()) { + throw new BufferUnderflowException(); + } + + return new NamesMessage(id, nameDataList); + + } catch (TransformationException e) { + throw new MessageException(e.getMessage(), e); + } + } + + public NamesMessage cloneWithNewId(int newId) { + NamesMessage clone = new NamesMessage(this.nameDataList); + clone.setId(newId); + return clone; + } + +} diff --git a/src/main/java/org/qortal/network/message/TransactionsMessage.java b/src/main/java/org/qortal/network/message/TransactionsMessage.java new file mode 100644 index 00000000..d7d60331 --- /dev/null +++ b/src/main/java/org/qortal/network/message/TransactionsMessage.java @@ -0,0 +1,76 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import org.qortal.data.transaction.TransactionData; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.TransactionTransformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class TransactionsMessage extends Message { + + private List transactions; + + public TransactionsMessage(List transactions) throws MessageException { + super(MessageType.TRANSACTIONS); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(transactions.size())); + + for (int i = 0; i < transactions.size(); ++i) { + TransactionData transactionData = transactions.get(i); + + byte[] serializedTransactionData = TransactionTransformer.toBytes(transactionData); + bytes.write(serializedTransactionData); + } + + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } catch (TransformationException e) { + throw new MessageException(e.getMessage(), e); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private TransactionsMessage(int id, List transactions) { + super(id, MessageType.TRANSACTIONS); + + this.transactions = transactions; + } + + public List getTransactions() { + return this.transactions; + } + + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { + try { + final int transactionCount = byteBuffer.getInt(); + + List transactions = new ArrayList<>(); + + for (int i = 0; i < transactionCount; ++i) { + TransactionData transactionData = TransactionTransformer.fromByteBuffer(byteBuffer); + transactions.add(transactionData); + } + + if (byteBuffer.hasRemaining()) { + throw new BufferUnderflowException(); + } + + return new TransactionsMessage(id, transactions); + + } catch (TransformationException e) { + throw new MessageException(e.getMessage(), e); + } + } + +} diff --git a/src/main/java/org/qortal/network/task/ChannelAcceptTask.java b/src/main/java/org/qortal/network/task/ChannelAcceptTask.java index 3e2a3033..da04cf9a 100644 --- a/src/main/java/org/qortal/network/task/ChannelAcceptTask.java +++ b/src/main/java/org/qortal/network/task/ChannelAcceptTask.java @@ -2,6 +2,7 @@ package org.qortal.network.task; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.arbitrary.ArbitraryDataFileManager; import org.qortal.network.Network; import org.qortal.network.Peer; import org.qortal.network.PeerAddress; @@ -65,6 +66,47 @@ public class ChannelAcceptTask implements Task { return; } + // We allow up to a maximum of maxPeers connected peers, of which... + // - maxDataPeers must be prearranged data connections (these are intentionally short-lived) + // - the remainder can be any regular peers + + // Firstly, determine the maximum limits + int maxPeers = Settings.getInstance().getMaxPeers(); + int maxDataPeers = Settings.getInstance().getMaxDataPeers(); + int maxRegularPeers = maxPeers - maxDataPeers; + + // Next, obtain the current state + int connectedDataPeerCount = Network.getInstance().getImmutableConnectedDataPeers().size(); + int connectedRegularPeerCount = Network.getInstance().getImmutableConnectedNonDataPeers().size(); + + // Check if the incoming connection should be considered a data or regular peer + boolean isDataPeer = ArbitraryDataFileManager.getInstance().isPeerRequestingData(address.getHost()); + + // Finally, decide if we have any capacity for this incoming peer + boolean connectionLimitReached; + if (isDataPeer) { + connectionLimitReached = (connectedDataPeerCount >= maxDataPeers); + } + else { + connectionLimitReached = (connectedRegularPeerCount >= maxRegularPeers); + } + + // Extra maxPeers check just to be safe + if (Network.getInstance().getImmutableConnectedPeers().size() >= maxPeers) { + connectionLimitReached = true; + } + + if (connectionLimitReached) { + try { + // We have enough peers + LOGGER.debug("Connection discarded from peer {} because the server is full", address); + socketChannel.close(); + } catch (IOException e) { + // IGNORE + } + return; + } + final Long now = NTP.getTime(); Peer newPeer; @@ -78,6 +120,10 @@ public class ChannelAcceptTask implements Task { LOGGER.debug("Connection accepted from peer {}", address); newPeer = new Peer(socketChannel); + if (isDataPeer) { + newPeer.setMaxConnectionAge(Settings.getInstance().getMaxDataPeerConnectionTime() * 1000L); + } + newPeer.setIsDataPeer(isDataPeer); network.addConnectedPeer(newPeer); } catch (IOException e) { diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index 714ada28..0d9325b9 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -62,6 +62,11 @@ public abstract class RepositoryManager { } public static boolean archive(Repository repository) { + if (Settings.getInstance().isLite()) { + // Lite nodes have no blockchain + return false; + } + // Bulk archive the database the first time we use archive mode if (Settings.getInstance().isArchiveEnabled()) { if (RepositoryManager.canArchiveOrPrune()) { @@ -82,6 +87,11 @@ public abstract class RepositoryManager { } public static boolean prune(Repository repository) { + if (Settings.getInstance().isLite()) { + // Lite nodes have no blockchain + return false; + } + // Bulk prune the database the first time we use top-only or block archive mode if (Settings.getInstance().isTopOnly() || Settings.getInstance().isArchiveEnabled()) { diff --git a/src/main/java/org/qortal/repository/TransactionRepository.java b/src/main/java/org/qortal/repository/TransactionRepository.java index 20096eb8..4fb9bb12 100644 --- a/src/main/java/org/qortal/repository/TransactionRepository.java +++ b/src/main/java/org/qortal/repository/TransactionRepository.java @@ -257,7 +257,8 @@ public interface TransactionRepository { * @return list of transactions, or empty if none. * @throws DataException */ - public List getUnconfirmedTransactions(Integer limit, Integer offset, Boolean reverse) throws DataException; + public List getUnconfirmedTransactions(List txTypes, byte[] creatorPublicKey, + Integer limit, Integer offset, Boolean reverse) throws DataException; /** * Returns list of unconfirmed transactions in timestamp-else-signature order. @@ -266,7 +267,7 @@ public interface TransactionRepository { * @throws DataException */ public default List getUnconfirmedTransactions() throws DataException { - return getUnconfirmedTransactions(null, null, null); + return getUnconfirmedTransactions(null, null, null, null, null); } /** diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index f228944e..e3ef13be 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -313,7 +313,7 @@ public class HSQLDBTransactionRepository implements TransactionRepository { @Override public List getSignaturesInvolvingAddress(String address) throws DataException { - String sql = "SELECT signature FROM TransactionRecipients WHERE participant = ?"; + String sql = "SELECT signature FROM TransactionParticipants WHERE participant = ?"; List signatures = new ArrayList<>(); @@ -1213,11 +1213,56 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } @Override - public List getUnconfirmedTransactions(Integer limit, Integer offset, Boolean reverse) throws DataException { - StringBuilder sql = new StringBuilder(256); - sql.append("SELECT signature FROM UnconfirmedTransactions "); + public List getUnconfirmedTransactions(List txTypes, byte[] creatorPublicKey, + Integer limit, Integer offset, Boolean reverse) throws DataException { + List whereClauses = new ArrayList<>(); + List bindParams = new ArrayList<>(); - sql.append("ORDER BY created_when"); + boolean hasCreatorPublicKey = creatorPublicKey != null; + boolean hasTxTypes = txTypes != null && !txTypes.isEmpty(); + + if (creatorPublicKey != null) { + whereClauses.add("Transactions.creator = ?"); + bindParams.add(creatorPublicKey); + } + + StringBuilder sql = new StringBuilder(256); + sql.append("SELECT signature FROM UnconfirmedTransactions"); + if (hasCreatorPublicKey || hasTxTypes) { + sql.append(" JOIN Transactions USING (signature) "); + } + + if (hasTxTypes) { + StringBuilder txTypesIn = new StringBuilder(256); + txTypesIn.append("Transactions.type IN ("); + + // ints are safe enough to use literally + final int txTypesSize = txTypes.size(); + for (int tti = 0; tti < txTypesSize; ++tti) { + if (tti != 0) + txTypesIn.append(", "); + + txTypesIn.append(txTypes.get(tti).value); + } + + txTypesIn.append(")"); + + whereClauses.add(txTypesIn.toString()); + } + + if (!whereClauses.isEmpty()) { + sql.append(" WHERE "); + + final int whereClausesSize = whereClauses.size(); + for (int wci = 0; wci < whereClausesSize; ++wci) { + if (wci != 0) + sql.append(" AND "); + + sql.append(whereClauses.get(wci)); + } + } + + sql.append(" ORDER BY created_when"); if (reverse != null && reverse) sql.append(" DESC"); @@ -1230,7 +1275,7 @@ public class HSQLDBTransactionRepository implements TransactionRepository { List transactions = new ArrayList<>(); // Find transactions with no corresponding row in BlockTransactions - try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) { + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { if (resultSet == null) return transactions; diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 252678d8..b58482b2 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -146,6 +146,8 @@ public class Settings { * This has a significant effect on execution time. */ private int onlineSignaturesTrimBatchSize = 100; // blocks + /** Lite nodes don't sync blocks, and instead request "derived data" from peers */ + private boolean lite = false; /** Whether we should prune old data to reduce database size * This prevents the node from being able to serve older blocks */ @@ -191,7 +193,9 @@ public class Settings { /** Target number of outbound connections to peers we should make. */ private int minOutboundPeers = 16; /** Maximum number of peer connections we allow. */ - private int maxPeers = 32; + private int maxPeers = 36; + /** Number of slots to reserve for short-lived QDN data transfers */ + private int maxDataPeers = 4; /** Maximum number of threads for network engine. */ private int maxNetworkThreadPoolSize = 32; /** Maximum number of threads for network proof-of-work compute, used during handshaking. */ @@ -210,6 +214,8 @@ public class Settings { private int minPeerConnectionTime = 5 * 60; // seconds /** Maximum time (in seconds) that we should attempt to remain connected to a peer for */ private int maxPeerConnectionTime = 60 * 60; // seconds + /** Maximum time (in seconds) that a peer should remain connected when requesting QDN data */ + private int maxDataPeerConnectionTime = 2 * 60; // seconds /** Whether to sync multiple blocks at once in normal operation */ private boolean fastSyncEnabled = true; @@ -655,6 +661,10 @@ public class Settings { return this.maxPeers; } + public int getMaxDataPeers() { + return this.maxDataPeers; + } + public int getMaxNetworkThreadPoolSize() { return this.maxNetworkThreadPoolSize; } @@ -673,6 +683,10 @@ public class Settings { public int getMaxPeerConnectionTime() { return this.maxPeerConnectionTime; } + public int getMaxDataPeerConnectionTime() { + return this.maxDataPeerConnectionTime; + } + public String getBlockchainConfig() { return this.blockchainConfig; } @@ -821,6 +835,10 @@ public class Settings { return this.onlineSignaturesTrimBatchSize; } + public boolean isLite() { + return this.lite; + } + public boolean isTopOnly() { return this.topOnly; } diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index 79a6478b..a1fd6baa 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -393,7 +393,10 @@ public abstract class Transaction { * @return transaction version number */ public static int getVersionByTimestamp(long timestamp) { - if (timestamp >= BlockChain.getInstance().getTransactionV5Timestamp()) { + if (timestamp >= BlockChain.getInstance().getTransactionV6Timestamp()) { + return 6; + } + else if (timestamp >= BlockChain.getInstance().getTransactionV5Timestamp()) { return 5; } return 4; @@ -530,11 +533,6 @@ public abstract class Transaction { if (now >= this.getDeadline()) return ValidationResult.TIMESTAMP_TOO_OLD; - // Transactions with a expiry prior to latest block's timestamp are too old - BlockData latestBlock = repository.getBlockRepository().getLastBlock(); - if (this.getDeadline() <= latestBlock.getTimestamp()) - return ValidationResult.TIMESTAMP_TOO_OLD; - // Transactions with a timestamp too far into future are too new long maxTimestamp = now + Settings.getInstance().getMaxTransactionTimestampFuture(); if (this.transactionData.getTimestamp() > maxTimestamp) @@ -545,6 +543,15 @@ public abstract class Transaction { if (feeValidationResult != ValidationResult.OK) return feeValidationResult; + if (Settings.getInstance().isLite()) { + // Everything from this point is difficult to validate for a lite node, since it has no blocks. + // For now, we will assume it is valid, to allow it to move around the network easily. + // If it turns out to be invalid, other full/top-only nodes will reject it on receipt. + // Lite nodes would never mint a block, so there's not much risk of holding invalid transactions. + // TODO: implement lite-only validation for each transaction type + return ValidationResult.OK; + } + PublicKeyAccount creator = this.getCreator(); if (creator == null) return ValidationResult.MISSING_CREATOR; @@ -553,6 +560,12 @@ public abstract class Transaction { if (countUnconfirmedByCreator(creator) >= Settings.getInstance().getMaxUnconfirmedPerAccount()) return ValidationResult.TOO_MANY_UNCONFIRMED; + // Transactions with a expiry prior to latest block's timestamp are too old + // Not relevant for lite nodes, as they don't have any blocks + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + if (this.getDeadline() <= latestBlock.getTimestamp()) + return ValidationResult.TIMESTAMP_TOO_OLD; + // Check transaction's txGroupId if (!this.isValidTxGroupId()) return ValidationResult.INVALID_TX_GROUP_ID; diff --git a/src/main/java/org/qortal/transform/transaction/AtTransactionTransformer.java b/src/main/java/org/qortal/transform/transaction/AtTransactionTransformer.java index 6b3c93f0..bb1381b2 100644 --- a/src/main/java/org/qortal/transform/transaction/AtTransactionTransformer.java +++ b/src/main/java/org/qortal/transform/transaction/AtTransactionTransformer.java @@ -4,8 +4,12 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import org.qortal.account.NullAccount; import org.qortal.data.transaction.ATTransactionData; +import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.transaction.Transaction; import org.qortal.transform.TransformationException; import org.qortal.utils.Serialization; @@ -17,12 +21,97 @@ public class AtTransactionTransformer extends TransactionTransformer { protected static final TransactionLayout layout = null; // Property lengths + + private static final int MESSAGE_SIZE_LENGTH = INT_LENGTH; + private static final int TYPE_LENGTH = INT_LENGTH; + + public static TransactionData fromByteBuffer(ByteBuffer byteBuffer) throws TransformationException { - throw new TransformationException("Serialized AT transactions should not exist!"); + long timestamp = byteBuffer.getLong(); + + int version = Transaction.getVersionByTimestamp(timestamp); + + byte[] reference = new byte[REFERENCE_LENGTH]; + byteBuffer.get(reference); + + String atAddress = Serialization.deserializeAddress(byteBuffer); + + String recipient = Serialization.deserializeAddress(byteBuffer); + + // Default to PAYMENT-type, as there were no MESSAGE-type transactions before transaction v6 + boolean isMessageType = false; + + if (version >= 6) { + // Version 6 supports both PAYMENT-type and MESSAGE-type, specified using an integer. + // This could be extended to support additional types at a later date, simply by adding + // additional integer values. + int type = byteBuffer.getInt(); + isMessageType = (type == 1); + } + + int messageLength = 0; + byte[] message = null; + long assetId = 0L; + long amount = 0L; + + if (isMessageType) { + messageLength = byteBuffer.getInt(); + + message = new byte[messageLength]; + byteBuffer.get(message); + } + else { + // Assume PAYMENT-type, as there were no MESSAGE-type transactions until this time + assetId = byteBuffer.getLong(); + + amount = byteBuffer.getLong(); + } + + long fee = byteBuffer.getLong(); + + byte[] signature = new byte[SIGNATURE_LENGTH]; + byteBuffer.get(signature); + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, NullAccount.PUBLIC_KEY, fee, signature); + + if (isMessageType) { + // MESSAGE-type + return new ATTransactionData(baseTransactionData, atAddress, recipient, message); + } + else { + // PAYMENT-type + return new ATTransactionData(baseTransactionData, atAddress, recipient, amount, assetId); + } + } public static int getDataLength(TransactionData transactionData) throws TransformationException { - throw new TransformationException("Serialized AT transactions should not exist!"); + ATTransactionData atTransactionData = (ATTransactionData) transactionData; + int version = Transaction.getVersionByTimestamp(transactionData.getTimestamp()); + + final int baseLength = TYPE_LENGTH + TIMESTAMP_LENGTH + REFERENCE_LENGTH + ADDRESS_LENGTH + ADDRESS_LENGTH + + FEE_LENGTH + SIGNATURE_LENGTH; + + int typeSpecificLength = 0; + + byte[] message = atTransactionData.getMessage(); + boolean isMessageType = (message != null); + + // MESSAGE-type and PAYMENT-type transactions will have differing lengths + if (isMessageType) { + typeSpecificLength = MESSAGE_SIZE_LENGTH + message.length; + } + else { + typeSpecificLength = ASSET_ID_LENGTH + AMOUNT_LENGTH; + } + + // V6 transactions include an extra integer to denote the type + int versionSpecificLength = 0; + if (version >= 6) { + versionSpecificLength = TYPE_LENGTH; + } + + return baseLength + typeSpecificLength + versionSpecificLength; } // Used for generating fake transaction signatures @@ -30,6 +119,8 @@ public class AtTransactionTransformer extends TransactionTransformer { try { ATTransactionData atTransactionData = (ATTransactionData) transactionData; + int version = Transaction.getVersionByTimestamp(atTransactionData.getTimestamp()); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); bytes.write(Ints.toByteArray(atTransactionData.getType().value)); @@ -42,7 +133,17 @@ public class AtTransactionTransformer extends TransactionTransformer { byte[] message = atTransactionData.getMessage(); - if (message != null) { + boolean isMessageType = (message != null); + int type = isMessageType ? 1 : 0; + + if (version >= 6) { + // Version 6 supports both PAYMENT-type and MESSAGE-type, specified using an integer. + // This could be extended to support additional types at a later date, simply by adding + // additional integer values. + bytes.write(Ints.toByteArray(type)); + } + + if (isMessageType) { // MESSAGE-type bytes.write(Ints.toByteArray(message.length)); bytes.write(message); diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 1f20ccfe..c8502d1b 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -58,7 +58,8 @@ "newBlockSigHeight": 320000, "shareBinFix": 399000, "calcChainWeightTimestamp": 1620579600000, - "transactionV5Timestamp": 1642176000000 + "transactionV5Timestamp": 1642176000000, + "transactionV6Timestamp": 9999999999999 }, "genesisInfo": { "version": 4, diff --git a/src/main/resources/i18n/SysTray_de.properties b/src/main/resources/i18n/SysTray_de.properties index 1880aa27..b949ca8c 100644 --- a/src/main/resources/i18n/SysTray_de.properties +++ b/src/main/resources/i18n/SysTray_de.properties @@ -27,6 +27,8 @@ DB_MAINTENANCE = Datenbank Instandhaltung EXIT = Verlassen +LITE_NODE = Lite node + MINTING_DISABLED = NOT minting MINTING_ENABLED = \u2714 Minting diff --git a/src/main/resources/i18n/SysTray_en.properties b/src/main/resources/i18n/SysTray_en.properties index 296f9760..204f0df2 100644 --- a/src/main/resources/i18n/SysTray_en.properties +++ b/src/main/resources/i18n/SysTray_en.properties @@ -27,6 +27,8 @@ DB_MAINTENANCE = Database Maintenance EXIT = Exit +LITE_NODE = Lite node + MINTING_DISABLED = NOT minting MINTING_ENABLED = \u2714 Minting diff --git a/src/main/resources/i18n/SysTray_fi.properties b/src/main/resources/i18n/SysTray_fi.properties index 07289551..bc787715 100644 --- a/src/main/resources/i18n/SysTray_fi.properties +++ b/src/main/resources/i18n/SysTray_fi.properties @@ -27,6 +27,8 @@ DB_MAINTENANCE = Tietokannan ylläpito EXIT = Pois +LITE_NODE = Lite node + MINTING_DISABLED = EI lyö rahaa MINTING_ENABLED = \u2714 Lyö rahaa diff --git a/src/main/resources/i18n/SysTray_fr.properties b/src/main/resources/i18n/SysTray_fr.properties index 61e8002a..6e60713c 100644 --- a/src/main/resources/i18n/SysTray_fr.properties +++ b/src/main/resources/i18n/SysTray_fr.properties @@ -27,6 +27,8 @@ DB_MAINTENANCE = Maintenance de la base de données EXIT = Quitter +LITE_NODE = Lite node + MINTING_DISABLED = NE mint PAS MINTING_ENABLED = \u2714 Minting diff --git a/src/main/resources/i18n/SysTray_hu.properties b/src/main/resources/i18n/SysTray_hu.properties index 5a082b6d..9bc51ff5 100644 --- a/src/main/resources/i18n/SysTray_hu.properties +++ b/src/main/resources/i18n/SysTray_hu.properties @@ -27,6 +27,8 @@ DB_MAINTENANCE = Adatbázis karbantartás EXIT = Kilépés +LITE_NODE = Lite node + MINTING_DISABLED = QORT-érmeverés jelenleg nincs folyamatban MINTING_ENABLED = \u2714 QORT-érmeverés folyamatban diff --git a/src/main/resources/i18n/SysTray_it.properties b/src/main/resources/i18n/SysTray_it.properties index dd02aefa..bf61cc46 100644 --- a/src/main/resources/i18n/SysTray_it.properties +++ b/src/main/resources/i18n/SysTray_it.properties @@ -27,6 +27,8 @@ DB_MAINTENANCE = Manutenzione del database EXIT = Uscita +LITE_NODE = Lite node + MINTING_DISABLED = Conio disabilitato MINTING_ENABLED = \u2714 Conio abilitato diff --git a/src/main/resources/i18n/SysTray_nl.properties b/src/main/resources/i18n/SysTray_nl.properties index 996c946f..8a4f112b 100644 --- a/src/main/resources/i18n/SysTray_nl.properties +++ b/src/main/resources/i18n/SysTray_nl.properties @@ -27,6 +27,8 @@ DB_MAINTENANCE = Database Onderhoud EXIT = Verlaten +LITE_NODE = Lite node + MINTING_DISABLED = Minten is uitgeschakeld MINTING_ENABLED = \u2714 Minten is ingeschakeld diff --git a/src/main/resources/i18n/SysTray_ru.properties b/src/main/resources/i18n/SysTray_ru.properties index efcc723c..fc3d8648 100644 --- a/src/main/resources/i18n/SysTray_ru.properties +++ b/src/main/resources/i18n/SysTray_ru.properties @@ -27,6 +27,8 @@ DB_MAINTENANCE = Обслуживание базы данных EXIT = Выход +LITE_NODE = Lite node + MINTING_DISABLED = Чеканка отключена MINTING_ENABLED = \u2714 Чеканка активна diff --git a/src/main/resources/i18n/SysTray_zh_CN.properties b/src/main/resources/i18n/SysTray_zh_CN.properties index 1216368b..c103d24b 100644 --- a/src/main/resources/i18n/SysTray_zh_CN.properties +++ b/src/main/resources/i18n/SysTray_zh_CN.properties @@ -27,6 +27,8 @@ DB_MAINTENANCE = 数据库维护 EXIT = 退出核心 +LITE_NODE = Lite node + MINTING_DISABLED = 没有铸币 MINTING_ENABLED = \u2714 铸币 diff --git a/src/main/resources/i18n/SysTray_zh_TW.properties b/src/main/resources/i18n/SysTray_zh_TW.properties index 1b2d0d90..5e6ccc3e 100644 --- a/src/main/resources/i18n/SysTray_zh_TW.properties +++ b/src/main/resources/i18n/SysTray_zh_TW.properties @@ -27,6 +27,8 @@ DB_MAINTENANCE = 數據庫維護 EXIT = 退出核心 +LITE_NODE = Lite node + MINTING_DISABLED = 沒有鑄幣 MINTING_ENABLED = \u2714 鑄幣 diff --git a/src/test/java/org/qortal/test/SerializationTests.java b/src/test/java/org/qortal/test/SerializationTests.java index 5a9fffa8..d9fe978c 100644 --- a/src/test/java/org/qortal/test/SerializationTests.java +++ b/src/test/java/org/qortal/test/SerializationTests.java @@ -47,7 +47,6 @@ public class SerializationTests extends Common { switch (txType) { case GENESIS: case ACCOUNT_FLAGS: - case AT: case CHAT: case PUBLICIZE: case AIRDROP: diff --git a/src/test/java/org/qortal/test/api/TransactionsApiTests.java b/src/test/java/org/qortal/test/api/TransactionsApiTests.java index 2cdd746b..102cac34 100644 --- a/src/test/java/org/qortal/test/api/TransactionsApiTests.java +++ b/src/test/java/org/qortal/test/api/TransactionsApiTests.java @@ -36,8 +36,8 @@ public class TransactionsApiTests extends ApiCommon { @Test public void testGetUnconfirmedTransactions() { - assertNotNull(this.transactionsResource.getUnconfirmedTransactions(null, null, null)); - assertNotNull(this.transactionsResource.getUnconfirmedTransactions(1, 1, true)); + assertNotNull(this.transactionsResource.getUnconfirmedTransactions(null, null, null, null, null)); + assertNotNull(this.transactionsResource.getUnconfirmedTransactions(null, null, 1, 1, true)); } @Test diff --git a/src/test/java/org/qortal/test/apps/CheckTranslations.java b/src/test/java/org/qortal/test/apps/CheckTranslations.java index 2b59ce84..b8008c78 100644 --- a/src/test/java/org/qortal/test/apps/CheckTranslations.java +++ b/src/test/java/org/qortal/test/apps/CheckTranslations.java @@ -15,7 +15,7 @@ public class CheckTranslations { private static final String[] SUPPORTED_LANGS = new String[] { "en", "de", "zh", "ru" }; private static final Set SYSTRAY_KEYS = Set.of("AUTO_UPDATE", "APPLYING_UPDATE_AND_RESTARTING", "BLOCK_HEIGHT", "BUILD_VERSION", "CHECK_TIME_ACCURACY", "CONNECTING", "CONNECTION", "CONNECTIONS", "CREATING_BACKUP_OF_DB_FILES", - "DB_BACKUP", "DB_CHECKPOINT", "EXIT", "MINTING_DISABLED", "MINTING_ENABLED", "OPEN_UI", "PERFORMING_DB_CHECKPOINT", + "DB_BACKUP", "DB_CHECKPOINT", "EXIT", "LITE_NODE", "MINTING_DISABLED", "MINTING_ENABLED", "OPEN_UI", "PERFORMING_DB_CHECKPOINT", "SYNCHRONIZE_CLOCK", "SYNCHRONIZING_BLOCKCHAIN", "SYNCHRONIZING_CLOCK"); private static String failurePrefix; diff --git a/src/test/java/org/qortal/test/at/AtSerializationTests.java b/src/test/java/org/qortal/test/at/AtSerializationTests.java new file mode 100644 index 00000000..3953bcdf --- /dev/null +++ b/src/test/java/org/qortal/test/at/AtSerializationTests.java @@ -0,0 +1,96 @@ +package org.qortal.test.at; + +import com.google.common.hash.HashCode; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.data.transaction.ATTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.Common; +import org.qortal.test.common.transaction.AtTestTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Base58; + +import static org.junit.Assert.assertEquals; + +public class AtSerializationTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @After + public void afterTest() throws DataException { + Common.orphanCheck(); + } + + + @Test + public void testPaymentTypeAtSerialization() throws DataException, TransformationException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Build PAYMENT-type AT transaction + PrivateKeyAccount signingAccount = Common.getTestAccount(repository, "alice"); + ATTransactionData transactionData = (ATTransactionData) AtTestTransaction.paymentType(repository, signingAccount, true); + Transaction transaction = Transaction.fromData(repository, transactionData); + transaction.sign(signingAccount); + + final int claimedLength = TransactionTransformer.getDataLength(transactionData); + byte[] serializedTransaction = TransactionTransformer.toBytes(transactionData); + assertEquals("Serialized PAYMENT-type AT transaction length differs from declared length", claimedLength, serializedTransaction.length); + + TransactionData deserializedTransactionData = TransactionTransformer.fromBytes(serializedTransaction); + // Re-sign + Transaction deserializedTransaction = Transaction.fromData(repository, deserializedTransactionData); + deserializedTransaction.sign(signingAccount); + assertEquals("Deserialized PAYMENT-type AT transaction signature differs", Base58.encode(transactionData.getSignature()), Base58.encode(deserializedTransactionData.getSignature())); + + // Re-serialize to check new length and bytes + final int reclaimedLength = TransactionTransformer.getDataLength(deserializedTransactionData); + assertEquals("Reserialized PAYMENT-type AT transaction declared length differs", claimedLength, reclaimedLength); + + byte[] reserializedTransaction = TransactionTransformer.toBytes(deserializedTransactionData); + assertEquals("Reserialized PAYMENT-type AT transaction bytes differ", HashCode.fromBytes(serializedTransaction).toString(), HashCode.fromBytes(reserializedTransaction).toString()); + } + } + + @Test + public void testMessageTypeAtSerialization() throws DataException, TransformationException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Build MESSAGE-type AT transaction + PrivateKeyAccount signingAccount = Common.getTestAccount(repository, "alice"); + ATTransactionData transactionData = (ATTransactionData) AtTestTransaction.messageType(repository, signingAccount, true); + Transaction transaction = Transaction.fromData(repository, transactionData); + transaction.sign(signingAccount); + + // MESSAGE-type AT transactions are only fully supported since transaction V6 + assertEquals(6, Transaction.getVersionByTimestamp(transactionData.getTimestamp())); + + final int claimedLength = TransactionTransformer.getDataLength(transactionData); + byte[] serializedTransaction = TransactionTransformer.toBytes(transactionData); + assertEquals("Serialized MESSAGE-type AT transaction length differs from declared length", claimedLength, serializedTransaction.length); + + TransactionData deserializedTransactionData = TransactionTransformer.fromBytes(serializedTransaction); + // Re-sign + Transaction deserializedTransaction = Transaction.fromData(repository, deserializedTransactionData); + deserializedTransaction.sign(signingAccount); + assertEquals("Deserialized MESSAGE-type AT transaction signature differs", Base58.encode(transactionData.getSignature()), Base58.encode(deserializedTransactionData.getSignature())); + + // Re-serialize to check new length and bytes + final int reclaimedLength = TransactionTransformer.getDataLength(deserializedTransactionData); + assertEquals("Reserialized MESSAGE-type AT transaction declared length differs", claimedLength, reclaimedLength); + + byte[] reserializedTransaction = TransactionTransformer.toBytes(deserializedTransactionData); + assertEquals("Reserialized MESSAGE-type AT transaction bytes differ", HashCode.fromBytes(serializedTransaction).toString(), HashCode.fromBytes(reserializedTransaction).toString()); + } + } + +} diff --git a/src/test/java/org/qortal/test/common/transaction/AtTestTransaction.java b/src/test/java/org/qortal/test/common/transaction/AtTestTransaction.java index 74216f2f..f827c396 100644 --- a/src/test/java/org/qortal/test/common/transaction/AtTestTransaction.java +++ b/src/test/java/org/qortal/test/common/transaction/AtTestTransaction.java @@ -12,16 +12,33 @@ import org.qortal.utils.Amounts; public class AtTestTransaction extends TestTransaction { public static TransactionData randomTransaction(Repository repository, PrivateKeyAccount account, boolean wantValid) throws DataException { + return AtTestTransaction.paymentType(repository, account, wantValid); + } + + public static TransactionData paymentType(Repository repository, PrivateKeyAccount account, boolean wantValid) throws DataException { byte[] signature = new byte[64]; random.nextBytes(signature); String atAddress = Crypto.toATAddress(signature); String recipient = account.getAddress(); + + // Use PAYMENT-type long amount = 123L * Amounts.MULTIPLIER; final long assetId = Asset.QORT; + + return new ATTransactionData(generateBase(account), atAddress, recipient, amount, assetId); + } + + public static TransactionData messageType(Repository repository, PrivateKeyAccount account, boolean wantValid) throws DataException { + byte[] signature = new byte[64]; + random.nextBytes(signature); + String atAddress = Crypto.toATAddress(signature); + String recipient = account.getAddress(); + + // Use MESSAGE-type byte[] message = new byte[32]; random.nextBytes(message); - return new ATTransactionData(generateBase(account), atAddress, recipient, amount, assetId, message); + return new ATTransactionData(generateBase(account), atAddress, recipient, message); } } diff --git a/src/test/resources/test-chain-v2-founder-rewards.json b/src/test/resources/test-chain-v2-founder-rewards.json index c0ea8fe5..e8a22dfc 100644 --- a/src/test/resources/test-chain-v2-founder-rewards.json +++ b/src/test/resources/test-chain-v2-founder-rewards.json @@ -52,7 +52,8 @@ "newBlockSigHeight": 999999, "shareBinFix": 999999, "calcChainWeightTimestamp": 0, - "transactionV5Timestamp": 0 + "transactionV5Timestamp": 0, + "transactionV6Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-leftover-reward.json b/src/test/resources/test-chain-v2-leftover-reward.json index 01505af0..eb6f56a9 100644 --- a/src/test/resources/test-chain-v2-leftover-reward.json +++ b/src/test/resources/test-chain-v2-leftover-reward.json @@ -52,7 +52,8 @@ "newBlockSigHeight": 999999, "shareBinFix": 999999, "calcChainWeightTimestamp": 0, - "transactionV5Timestamp": 0 + "transactionV5Timestamp": 0, + "transactionV6Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-minting.json b/src/test/resources/test-chain-v2-minting.json index fcabe4bf..d9893cb2 100644 --- a/src/test/resources/test-chain-v2-minting.json +++ b/src/test/resources/test-chain-v2-minting.json @@ -52,7 +52,8 @@ "newBlockSigHeight": 999999, "shareBinFix": 999999, "calcChainWeightTimestamp": 0, - "transactionV5Timestamp": 0 + "transactionV5Timestamp": 0, + "transactionV6Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-qora-holder-extremes.json b/src/test/resources/test-chain-v2-qora-holder-extremes.json index 8ec94631..3fbb1375 100644 --- a/src/test/resources/test-chain-v2-qora-holder-extremes.json +++ b/src/test/resources/test-chain-v2-qora-holder-extremes.json @@ -52,7 +52,8 @@ "newBlockSigHeight": 999999, "shareBinFix": 999999, "calcChainWeightTimestamp": 0, - "transactionV5Timestamp": 0 + "transactionV5Timestamp": 0, + "transactionV6Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-qora-holder.json b/src/test/resources/test-chain-v2-qora-holder.json index 38a563b2..b14dc8b7 100644 --- a/src/test/resources/test-chain-v2-qora-holder.json +++ b/src/test/resources/test-chain-v2-qora-holder.json @@ -52,7 +52,8 @@ "newBlockSigHeight": 999999, "shareBinFix": 999999, "calcChainWeightTimestamp": 0, - "transactionV5Timestamp": 0 + "transactionV5Timestamp": 0, + "transactionV6Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-reward-levels.json b/src/test/resources/test-chain-v2-reward-levels.json index ab934d26..932a8cda 100644 --- a/src/test/resources/test-chain-v2-reward-levels.json +++ b/src/test/resources/test-chain-v2-reward-levels.json @@ -52,7 +52,8 @@ "newBlockSigHeight": 999999, "shareBinFix": 6, "calcChainWeightTimestamp": 0, - "transactionV5Timestamp": 0 + "transactionV5Timestamp": 0, + "transactionV6Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-reward-scaling.json b/src/test/resources/test-chain-v2-reward-scaling.json index b3e358b2..6244ea58 100644 --- a/src/test/resources/test-chain-v2-reward-scaling.json +++ b/src/test/resources/test-chain-v2-reward-scaling.json @@ -52,7 +52,8 @@ "newBlockSigHeight": 999999, "shareBinFix": 999999, "calcChainWeightTimestamp": 0, - "transactionV5Timestamp": 0 + "transactionV5Timestamp": 0, + "transactionV6Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index 20ff391c..f308712d 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -52,7 +52,8 @@ "newBlockSigHeight": 999999, "shareBinFix": 999999, "calcChainWeightTimestamp": 0, - "transactionV5Timestamp": 0 + "transactionV5Timestamp": 0, + "transactionV6Timestamp": 0 }, "genesisInfo": { "version": 4,