diff --git a/TestNets.md b/TestNets.md index 6f8e92e6..e475e593 100644 --- a/TestNets.md +++ b/TestNets.md @@ -41,13 +41,39 @@ - Start up at least as many nodes as `minBlockchainPeers` (or adjust this value instead) - Probably best to perform API call `DELETE /peers/known` - Add other nodes via API call `POST /peers ` -- Add minting private key to node(s) via API call `POST /admin/mintingaccounts ` - This key must have corresponding `REWARD_SHARE` transaction in testnet genesis block +- Add minting private key to nodes via API call `POST /admin/mintingaccounts ` + The keys must have corresponding `REWARD_SHARE` transactions in testnet genesis block +- You must have at least 2 separate minting keys and two separate nodes. Assign one minting key to each node. +- Alternatively, comment out the `if (mintedLastBlock) { }` conditional in BlockMinter.java to allow for a single node and key. - Wait for genesis block timestamp to pass - A node should mint block 2 approximately 60 seconds after genesis block timestamp - Other testnet nodes will sync *as long as there is at least `minBlockchainPeers` peers with an "up-to-date" chain` - You can also use API call `POST /admin/forcesync ` on stuck nodes +## Single-node testnet + +A single-node testnet is possible with code modifications, for basic testing, or to more easily start a new testnet. +To do so, follow these steps: +- Comment out the `if (mintedLastBlock) { }` conditional in BlockMinter.java +- Comment out the `minBlockchainPeers` validation in Settings.validate() +- Set `minBlockchainPeers` to 0 in settings.json +- Set `Synchronizer.RECOVERY_MODE_TIMEOUT` to `0` +- All other steps should remain the same. Only a single reward share key is needed. +- Remember to put these values back after introducing other nodes + +## Fixed network + +To restrict a testnet to a set of private nodes, you can use the "fixed network" feature. +This ensures that the testnet nodes only communicate with each other and not other known peers. +To do this, add the following setting to each testnet node, substituting the IP addresses: +``` +"fixedNetwork": [ + "192.168.0.101:62392", + "192.168.0.102:62392", + "192.168.0.103:62392" +] +``` + ## Dealing with stuck chain Maybe your nodes have been offline and no-one has minted a recent testnet block. diff --git a/WindowsInstaller/Qortal.aip b/WindowsInstaller/Qortal.aip index 59da9519..f69f0682 100755 --- a/WindowsInstaller/Qortal.aip +++ b/WindowsInstaller/Qortal.aip @@ -17,10 +17,10 @@ - + - + @@ -212,7 +212,7 @@ - + diff --git a/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.jar b/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.jar new file mode 100644 index 00000000..10a92c07 Binary files /dev/null and b/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.jar differ diff --git a/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom b/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom new file mode 100644 index 00000000..96e0fe50 --- /dev/null +++ b/lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom @@ -0,0 +1,9 @@ + + + 4.0.0 + com.dosse + WaifUPnP + 1.1 + POM was created from install:install-file + diff --git a/lib/com/dosse/WaifUPnP/maven-metadata-local.xml b/lib/com/dosse/WaifUPnP/maven-metadata-local.xml new file mode 100644 index 00000000..07d6ffd0 --- /dev/null +++ b/lib/com/dosse/WaifUPnP/maven-metadata-local.xml @@ -0,0 +1,12 @@ + + + com.dosse + WaifUPnP + + 1.1 + + 1.1 + + 20220218200127 + + diff --git a/pom.xml b/pom.xml index 798d68ea..f6d68377 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.0.4 + 3.1.1 jar true @@ -21,6 +21,7 @@ 1.2.2 28.1-jre 2.5.1 + 1.1 2.29.1 9.4.29.v20200521 2.17.1 @@ -427,6 +428,12 @@ AT ${ciyam-at.version} + + + com.dosse + WaifUPnP + ${upnp.version} + org.bitcoinj diff --git a/src/main/java/org/qortal/account/Account.java b/src/main/java/org/qortal/account/Account.java index 417dde6d..aeff7810 100644 --- a/src/main/java/org/qortal/account/Account.java +++ b/src/main/java/org/qortal/account/Account.java @@ -272,7 +272,7 @@ public class Account { /** * Returns 'effective' minting level, or zero if reward-share does not exist. *

- * For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config. + * this is being used on src/main/java/org/qortal/api/resource/AddressesResource.java to fulfil the online accounts api call * * @param repository * @param rewardSharePublicKey @@ -288,5 +288,26 @@ public class Account { Account rewardShareMinter = new Account(repository, rewardShareData.getMinter()); return rewardShareMinter.getEffectiveMintingLevel(); } + /** + * Returns 'effective' minting level, with a fix for the zero level. + *

+ * For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config. + * + * @param repository + * @param rewardSharePublicKey + * @return 0+ + * @throws DataException + */ + public static int getRewardShareEffectiveMintingLevelIncludingLevelZero(Repository repository, byte[] rewardSharePublicKey) throws DataException { + // Find actual minter and get their effective minting level + RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey); + if (rewardShareData == null) + return 0; + else if(!rewardShareData.getMinter().equals(rewardShareData.getRecipient()))//the minter is different than the recipient this means sponsorship + return 0; + + Account rewardShareMinter = new Account(repository, rewardShareData.getMinter()); + return rewardShareMinter.getEffectiveMintingLevel(); + } } diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java index 474b6417..026d9210 100644 --- a/src/main/java/org/qortal/api/HTMLParser.java +++ b/src/main/java/org/qortal/api/HTMLParser.java @@ -28,6 +28,11 @@ public class HTMLParser { // Add base href tag String baseElement = String.format("", baseUrl); head.get(0).prepend(baseElement); + + // Add meta charset tag + String metaCharsetElement = ""; + head.get(0).prepend(metaCharsetElement); + } String html = document.html(); this.data = html.getBytes(); diff --git a/src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java b/src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java index 27770449..cc21587d 100644 --- a/src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java +++ b/src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java @@ -17,7 +17,7 @@ import java.util.Map; @Path("/") -@Tag(name = "Gateway") +@Tag(name = "Domain Map") public class DomainMapResource { @Context HttpServletRequest request; diff --git a/src/main/java/org/qortal/api/resource/AddressesResource.java b/src/main/java/org/qortal/api/resource/AddressesResource.java index abe1960c..39c76cf4 100644 --- a/src/main/java/org/qortal/api/resource/AddressesResource.java +++ b/src/main/java/org/qortal/api/resource/AddressesResource.java @@ -198,7 +198,7 @@ public class AddressesResource { for (OnlineAccountData onlineAccountData : onlineAccounts) { try { - final int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, onlineAccountData.getPublicKey()); + final int minterLevel = Account.getRewardShareEffectiveMintingLevelIncludingLevelZero(repository, onlineAccountData.getPublicKey()); OnlineAccountLevel onlineAccountLevel = onlineAccountLevels.stream() .filter(a -> a.getLevel() == minterLevel) diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index bde4bed4..7f4a3806 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -315,6 +315,7 @@ public class AdminResource { repository.getAccountRepository().save(mintingAccountData); repository.saveChanges(); + repository.exportNodeLocalData();//after adding new minting account let's persist it to the backup MintingAccounts.json } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e); } catch (DataException e) { @@ -355,6 +356,7 @@ public class AdminResource { return "false"; repository.saveChanges(); + repository.exportNodeLocalData();//after removing new minting account let's persist it to the backup MintingAccounts.json } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e); } catch (DataException e) { @@ -546,7 +548,7 @@ public class AdminResource { @Path("/repository/data") @Operation( summary = "Export sensitive/node-local data from repository.", - description = "Exports data to .script files on local machine" + description = "Exports data to .json files on local machine" ) @ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE}) @SecurityRequirement(name = "apiKey") diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 1b35ee27..7d331074 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -430,17 +430,12 @@ public class ArbitraryResource { @ApiErrors({ApiError.REPOSITORY_ISSUE}) public List getHostedTransactions(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @Parameter(ref = "limit") @QueryParam("limit") Integer limit, - @Parameter(ref = "offset") @QueryParam("offset") Integer offset, - @QueryParam("includemetadata") Boolean includeMetadata) { + @Parameter(ref = "offset") @QueryParam("offset") Integer offset) { Security.checkApiCallAllowed(request); - if (includeMetadata == null) { - includeMetadata = false; - } - try (final Repository repository = RepositoryManager.getRepository()) { - List hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset, includeMetadata); + List hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset); return hostedTransactions; @@ -465,18 +460,21 @@ public class ArbitraryResource { @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus, @Parameter(ref = "limit") @QueryParam("limit") Integer limit, @Parameter(ref = "offset") @QueryParam("offset") Integer offset, - @QueryParam("includemetadata") Boolean includeMetadata) { + @QueryParam("query") String query) { Security.checkApiCallAllowed(request); List resources = new ArrayList<>(); - if (includeMetadata == null) { - includeMetadata = false; - } - try (final Repository repository = RepositoryManager.getRepository()) { + + List transactionDataList; + + if (query == null || query.equals("")) { + transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset); + } else { + transactionDataList = ArbitraryDataStorageManager.getInstance().searchHostedTransactions(repository,query, limit, offset); + } - List transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset, includeMetadata); for (ArbitraryTransactionData transactionData : transactionDataList) { ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); arbitraryResourceInfo.name = transactionData.getName(); @@ -498,6 +496,8 @@ public class ArbitraryResource { } } + + @DELETE @Path("/resource/{service}/{name}/{identifier}") @Operation( diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java index 834c7b81..80d19804 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java @@ -122,7 +122,7 @@ public class CrossChainBitcoinResource { @Path("/send") @Operation( summary = "Sends BTC from hierarchical, deterministic BIP32 wallet to specific address", - description = "Currently only supports 'legacy' P2PKH Bitcoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", + description = "Currently supports 'legacy' P2PKH Bitcoin addresses and Native SegWit (P2WPKH) addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", requestBody = @RequestBody( required = true, content = @Content( diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java index 627c00c7..8ac0f9a0 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java @@ -122,7 +122,7 @@ public class CrossChainLitecoinResource { @Path("/send") @Operation( summary = "Sends LTC from hierarchical, deterministic BIP32 wallet to specific address", - description = "Currently only supports 'legacy' P2PKH Litecoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", + description = "Currently supports 'legacy' P2PKH Litecoin addresses and Native SegWit (P2WPKH) addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", requestBody = @RequestBody( required = true, content = @Content( diff --git a/src/main/java/org/qortal/api/resource/PeersResource.java b/src/main/java/org/qortal/api/resource/PeersResource.java index 97e2644e..38461141 100644 --- a/src/main/java/org/qortal/api/resource/PeersResource.java +++ b/src/main/java/org/qortal/api/resource/PeersResource.java @@ -354,7 +354,7 @@ public class PeersResource { List connectedPeers = Network.getInstance().getConnectedPeers().stream().collect(Collectors.toList()); for (Peer peer : connectedPeers) { - if (peer.isOutbound()) { + if (!peer.isOutbound()) { peersSummary.inboundConnections++; } else { diff --git a/src/main/java/org/qortal/api/resource/TransactionsResource.java b/src/main/java/org/qortal/api/resource/TransactionsResource.java index 9bc6d497..55ad7cde 100644 --- a/src/main/java/org/qortal/api/resource/TransactionsResource.java +++ b/src/main/java/org/qortal/api/resource/TransactionsResource.java @@ -638,7 +638,10 @@ public class TransactionsResource { ApiError.BLOCKCHAIN_NEEDS_SYNC, ApiError.INVALID_SIGNATURE, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE }) public String processTransaction(String rawBytes58) { - if (!Controller.getInstance().isUpToDate()) + // Only allow a transaction to be processed if our latest block is less than 30 minutes old + // If older than this, we should first wait until the blockchain is synced + final Long minLatestBlockTimestamp = NTP.getTime() - (30 * 60 * 1000L); + if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC); byte[] rawBytes = Base58.decode(rawBytes58); diff --git a/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java b/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java index 26d131c4..c579ac86 100644 --- a/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java @@ -20,6 +20,7 @@ import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; import org.eclipse.jetty.websocket.api.annotations.WebSocket; import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.crypto.Crypto; import org.qortal.data.transaction.PresenceTransactionData; import org.qortal.data.transaction.TransactionData; @@ -99,13 +100,13 @@ public class PresenceWebSocket extends ApiWebSocket implements Listener { @Override public void listen(Event event) { - // We use NewBlockEvent as a proxy for 1-minute timer - if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Controller.NewBlockEvent)) + // We use Synchronizer.NewChainTipEvent as a proxy for 1-minute timer + if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Synchronizer.NewChainTipEvent)) return; removeOldEntries(); - if (event instanceof Controller.NewBlockEvent) + if (event instanceof Synchronizer.NewChainTipEvent) // We only wanted a chance to cull old entries return; diff --git a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java index 186f79e3..35fc4691 100644 --- a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java @@ -23,6 +23,7 @@ import org.eclipse.jetty.websocket.api.annotations.WebSocket; import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; import org.qortal.api.model.CrossChainOfferSummary; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.crosschain.SupportedBlockchain; import org.qortal.crosschain.ACCT; import org.qortal.crosschain.AcctMode; @@ -80,10 +81,10 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener { @Override public void listen(Event event) { - if (!(event instanceof Controller.NewBlockEvent)) + if (!(event instanceof Synchronizer.NewChainTipEvent)) return; - BlockData blockData = ((Controller.NewBlockEvent) event).getBlockData(); + BlockData blockData = ((Synchronizer.NewChainTipEvent) event).getNewChainTip(); // Process any new info diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 026898df..4fd2ee2e 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -366,6 +366,21 @@ public class ArbitraryDataFile { return false; } + public boolean delete(int attempts) { + // Keep trying to delete the data until it is deleted, or we reach 10 attempts + for (int i=0; i excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); + + // Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts + List accountsToSend; + synchronized (this.onlineAccounts) { + accountsToSend = new ArrayList<>(this.onlineAccounts); + } + + Iterator iterator = accountsToSend.iterator(); + + SEND_ITERATOR: + while (iterator.hasNext()) { + OnlineAccountData onlineAccountData = iterator.next(); + + for (int i = 0; i < excludeAccounts.size(); ++i) { + OnlineAccountData excludeAccountData = excludeAccounts.get(i); + + if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) { + iterator.remove(); + continue SEND_ITERATOR; + } + } + } + + Message onlineAccountsMessage = new OnlineAccountsV2Message(accountsToSend); + peer.sendMessage(onlineAccountsMessage); + + LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer)); + } + + private void onNetworkOnlineAccountsV2Message(Peer peer, Message message) { + OnlineAccountsV2Message onlineAccountsMessage = (OnlineAccountsV2Message) message; + + List peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts(); + LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer)); + + try (final Repository repository = RepositoryManager.getRepository()) { + for (OnlineAccountData onlineAccountData : peersOnlineAccounts) + this.verifyAndAddAccount(repository, onlineAccountData); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e); + } + } + // Utilities private void verifyAndAddAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException { @@ -1815,11 +1872,17 @@ public class Controller extends Thread { // Request data from other peers? if ((this.onlineAccountsTasksTimestamp % ONLINE_ACCOUNTS_BROADCAST_INTERVAL) < ONLINE_ACCOUNTS_TASKS_INTERVAL) { - Message message; + List safeOnlineAccounts; synchronized (this.onlineAccounts) { - message = new GetOnlineAccountsMessage(this.onlineAccounts); + safeOnlineAccounts = new ArrayList<>(this.onlineAccounts); } - Network.getInstance().broadcast(peer -> message); + + Message messageV1 = new GetOnlineAccountsMessage(safeOnlineAccounts); + Message messageV2 = new GetOnlineAccountsV2Message(safeOnlineAccounts); + + Network.getInstance().broadcast(peer -> + peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1 + ); } // Refresh our online accounts signatures? @@ -1911,8 +1974,12 @@ public class Controller extends Thread { if (!hasInfoChanged) return; - Message message = new OnlineAccountsMessage(ourOnlineAccounts); - Network.getInstance().broadcast(peer -> message); + Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts); + Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts); + + Network.getInstance().broadcast(peer -> + peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1 + ); LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp)); } @@ -1998,10 +2065,13 @@ public class Controller extends Thread { return peers; } - /** Returns whether we think our node has up-to-date blockchain based on our info about other peers. */ - public boolean isUpToDate() { + /** + * Returns whether we think our node has up-to-date blockchain based on our info about other peers. + * @param minLatestBlockTimestamp - the minimum block timestamp to be considered recent + * @return boolean - whether our node's blockchain is up to date or not + */ + public boolean isUpToDate(Long minLatestBlockTimestamp) { // Do we even have a vaguely recent block? - final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp(); if (minLatestBlockTimestamp == null) return false; @@ -2027,6 +2097,16 @@ public class Controller extends Thread { return !peers.isEmpty(); } + /** + * Returns whether we think our node has up-to-date blockchain based on our info about other peers. + * Uses the default minLatestBlockTimestamp value. + * @return boolean - whether our node's blockchain is up to date or not + */ + public boolean isUpToDate() { + final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp(); + return this.isUpToDate(minLatestBlockTimestamp); + } + /** Returns minimum block timestamp for block to be considered 'recent', or null if NTP not synced. */ public static Long getMinimumLatestBlockTimestamp() { Long now = NTP.getTime(); diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index b98c5fa2..bb36d42d 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -22,6 +22,8 @@ import org.qortal.data.block.CommonBlockData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.transaction.RewardShareTransactionData; import org.qortal.data.transaction.TransactionData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; import org.qortal.network.Network; import org.qortal.network.Peer; import org.qortal.network.message.BlockMessage; @@ -96,6 +98,24 @@ public class Synchronizer extends Thread { OK, NOTHING_TO_DO, GENESIS_ONLY, NO_COMMON_BLOCK, TOO_DIVERGENT, NO_REPLY, INFERIOR_CHAIN, INVALID_DATA, NO_BLOCKCHAIN_LOCK, REPOSITORY_ISSUE, SHUTTING_DOWN; } + public static class NewChainTipEvent implements Event { + private final BlockData priorChainTip; + private final BlockData newChainTip; + + public NewChainTipEvent(BlockData priorChainTip, BlockData newChainTip) { + this.priorChainTip = priorChainTip; + this.newChainTip = newChainTip; + } + + public BlockData getPriorChainTip() { + return this.priorChainTip; + } + + public BlockData getNewChainTip() { + return this.newChainTip; + } + } + // Constructors private Synchronizer() { @@ -338,6 +358,8 @@ public class Synchronizer extends Thread { Network network = Network.getInstance(); network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip)); + + EventBus.INSTANCE.notify(new NewChainTipEvent(priorChainTip, newChainTip)); } return syncResult; diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index 64916df5..3edf0ef8 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -222,7 +222,11 @@ public class ArbitraryDataCleanupManager extends Thread { try (final Repository repository = RepositoryManager.getRepository()) { // Check if there are any hosted files that don't have matching transactions - this.checkForExpiredTransactions(repository); + // UPDATE: This has been disabled for now as it was deleting valid transactions + // and causing chunks to go missing on the network. If ever re-enabled, we MUST + // ensure that original copies of data aren't deleted, and that sufficient time + // is allowed (ideally several hours) before treating a transaction as missing. + // this.checkForExpiredTransactions(repository); // Delete additional data at random if we're over our storage limit // Use the DELETION_THRESHOLD so that we only start deleting once the hard limit is reached diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index bc5f5769..367806cd 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -29,6 +29,7 @@ public class ArbitraryDataFileListManager { private static ArbitraryDataFileListManager instance; + private static String MIN_PEER_VERSION_FOR_FILE_LIST_STATS = "3.2.0"; /** * Map of recent incoming requests for ARBITRARY transaction data file lists. @@ -266,18 +267,16 @@ public class ArbitraryDataFileListManager { List handshakedPeers = Network.getInstance().getHandshakedPeers(); List missingHashes = null; -// // TODO: uncomment after GetArbitraryDataFileListMessage updates are deployed -// // Find hashes that we are missing -// try { -// ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); -// arbitraryDataFile.setMetadataHash(metadataHash); -// missingHashes = arbitraryDataFile.missingHashes(); -// } catch (DataException e) { -// // Leave missingHashes as null, so that all hashes are requested -// } -// int hashCount = missingHashes != null ? missingHashes.size() : 0; + // Find hashes that we are missing + try { + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + missingHashes = arbitraryDataFile.missingHashes(); + } catch (DataException e) { + // Leave missingHashes as null, so that all hashes are requested + } + int hashCount = missingHashes != null ? missingHashes.size() : 0; - int hashCount = 0; LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size())); // Build request @@ -405,6 +404,13 @@ public class ArbitraryDataFileListManager { ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size()); + if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) { + long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime(); + LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}", + totalRequestTime, arbitraryDataFileListMessage.getRequestHops(), + arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); + } + // Do we have a pending request for this data? Triple request = arbitraryDataFileListRequests.get(message.getId()); if (request == null || request.getA() == null) { @@ -474,12 +480,26 @@ public class ArbitraryDataFileListManager { if (!isBlocked) { Peer requestingPeer = request.getB(); if (requestingPeer != null) { + Long requestTime = arbitraryDataFileListMessage.getRequestTime(); + Integer requestHops = arbitraryDataFileListMessage.getRequestHops(); + // Add each hash to our local mapping so we know who to ask later Long now = NTP.getTime(); for (byte[] hash : hashes) { String hash58 = Base58.encode(hash); - ArbitraryRelayInfo relayMap = new ArbitraryRelayInfo(hash58, signature58, peer, now); - ArbitraryDataFileManager.getInstance().addToRelayMap(relayMap); + ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops); + ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo); + } + + // Bump requestHops if it exists + if (requestHops != null) { + arbitraryDataFileListMessage.setRequestHops(++requestHops); + } + + // Remove optional parameters if the requesting peer doesn't support it yet + // A message with less statistical data is better than no message at all + if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { + arbitraryDataFileListMessage.removeOptionalStats(); } // Forward to requesting peer @@ -584,8 +604,17 @@ public class ArbitraryDataFileListManager { arbitraryDataFileListRequests.put(message.getId(), newEntry); } - ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + String ourAddress = Network.getInstance().getOurExternalIpAddress(); + ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, + hashes, NTP.getTime(), 0, ourAddress, true); arbitraryDataFileListMessage.setId(message.getId()); + + // Remove optional parameters if the requesting peer doesn't support it yet + // A message with less statistical data is better than no message at all + if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { + arbitraryDataFileListMessage.removeOptionalStats(); + } + if (!peer.sendMessage(arbitraryDataFileListMessage)) { LOGGER.debug("Couldn't send list of hashes"); peer.disconnect("failed to send list of hashes"); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index ff502ff4..e60e32a9 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -37,7 +37,7 @@ public class ArbitraryDataFileManager extends Thread { /** * Map to keep track of our in progress (outgoing) arbitrary data file requests */ - private Map arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>()); + public Map arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>()); /** * Map to keep track of hashes that we might need to relay @@ -148,7 +148,7 @@ public class ArbitraryDataFileManager extends Thread { } } else { - LOGGER.trace("Already requesting data file {} for signature {}", arbitraryDataFile, Base58.encode(signature)); + LOGGER.trace("Already requesting data file {} for signature {} from peer {}", arbitraryDataFile, Base58.encode(signature), peer); } } else { @@ -240,16 +240,7 @@ public class ArbitraryDataFileManager extends Thread { ArbitraryDataFile dataFile = arbitraryDataFileMessage.getArbitraryDataFile(); // Keep trying to delete the data until it is deleted, or we reach 10 attempts - for (int i=0; i<10; i++) { - if (dataFile.delete()) { - break; - } - try { - Thread.sleep(1000L); - } catch (InterruptedException e) { - // Fall through to exit method - } - } + dataFile.delete(10); } } @@ -401,6 +392,33 @@ public class ArbitraryDataFileManager extends Thread { } } + private ArbitraryRelayInfo getOptimalRelayInfoEntryForHash(String hash58) { + LOGGER.trace("Fetching relay info for hash: {}", hash58); + List relayInfoList = this.getRelayInfoListForHash(hash58); + if (relayInfoList != null && !relayInfoList.isEmpty()) { + + // Remove any with null requestHops + relayInfoList.removeIf(r -> r.getRequestHops() == null); + + // If list is now empty, then just return one at random + if (relayInfoList.isEmpty()) { + return this.getRandomRelayInfoEntryForHash(hash58); + } + + // Sort by number of hops (lowest first) + relayInfoList.sort(Comparator.comparingInt(ArbitraryRelayInfo::getRequestHops)); + + // FUTURE: secondary sort by requestTime? + + ArbitraryRelayInfo relayInfo = relayInfoList.get(0); + + LOGGER.trace("Returning optimal relay info for hash: {} (requestHops {})", hash58, relayInfo.getRequestHops()); + return relayInfo; + } + LOGGER.trace("No relay info exists for hash: {}", hash58); + return null; + } + private ArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) { LOGGER.trace("Fetching random relay info for hash: {}", hash58); List relayInfoList = this.getRelayInfoListForHash(hash58); @@ -451,7 +469,7 @@ public class ArbitraryDataFileManager extends Thread { try { ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); - ArbitraryRelayInfo relayInfo = this.getRandomRelayInfoEntryForHash(hash58); + ArbitraryRelayInfo relayInfo = this.getOptimalRelayInfoEntryForHash(hash58); if (arbitraryDataFile.exists()) { LOGGER.trace("Hash {} exists", hash58); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java index 5f491411..e46fd2fb 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -31,8 +31,6 @@ public class ArbitraryDataFileRequestThread implements Runnable { try { while (!Controller.isStopping()) { - Thread.sleep(1000); - Long now = NTP.getTime(); this.processFileHashes(now); } @@ -41,7 +39,7 @@ public class ArbitraryDataFileRequestThread implements Runnable { } } - private void processFileHashes(Long now) { + private void processFileHashes(Long now) throws InterruptedException { if (Controller.isStopping()) { return; } @@ -82,6 +80,12 @@ public class ArbitraryDataFileRequestThread implements Runnable { continue; } + // Skip if already requesting, but don't remove, as we might want to retry later + if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) { + // Already requesting - leave this attempt for later + continue; + } + // We want to process this file shouldProcess = true; iterator.remove(); @@ -91,6 +95,7 @@ public class ArbitraryDataFileRequestThread implements Runnable { if (!shouldProcess) { // Nothing to do + Thread.sleep(1000L); return; } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index ee67e435..6c3a32e6 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -38,7 +38,7 @@ public class ArbitraryDataManager extends Thread { private int powDifficulty = 14; // Must not be final, as unit tests need to reduce this value /** Request timeout when transferring arbitrary data */ - public static final long ARBITRARY_REQUEST_TIMEOUT = 10 * 1000L; // ms + public static final long ARBITRARY_REQUEST_TIMEOUT = 12 * 1000L; // ms /** Maximum time to hold information about an in-progress relay */ public static final long ARBITRARY_RELAY_TIMEOUT = 60 * 1000L; // ms @@ -80,6 +80,9 @@ public class ArbitraryDataManager extends Thread { Thread.currentThread().setName("Arbitrary Data Manager"); try { + // Wait for node to finish starting up and making connections + Thread.sleep(2 * 60 * 1000L); + while (!isStopping) { Thread.sleep(2000); @@ -370,7 +373,7 @@ public class ArbitraryDataManager extends Thread { public void broadcastHostedSignatureList() { try (final Repository repository = RepositoryManager.getRepository()) { - List hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, null, null, false); + List hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, null, null); List hostedSignatures = hostedTransactions.stream().map(ArbitraryTransactionData::getSignature).collect(Collectors.toList()); if (!hostedSignatures.isEmpty()) { // Broadcast the list, using null to represent our peer address diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index 76290dc0..2c992c61 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -16,6 +16,7 @@ import org.qortal.utils.FilesystemUtils; import org.qortal.utils.NTP; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -46,6 +47,9 @@ public class ArbitraryDataStorageManager extends Thread { private List hostedTransactions; + private String searchQuery; + private List searchResultsTransactions; + private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes /** Treat storage as full at 90% usage, to reduce risk of going over the limit. @@ -257,14 +261,8 @@ public class ArbitraryDataStorageManager extends Thread { } - // Hosted data - - public List listAllHostedTransactions(Repository repository, Integer limit, Integer offset, boolean includeMetadataOnly) { - // Load from cache if we can, to avoid disk reads - if (this.hostedTransactions != null) { - return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset); - } - + public List loadAllHostedTransactions(Repository repository) { + List arbitraryTransactionDataList = new ArrayList<>(); // Find all hosted paths @@ -287,15 +285,13 @@ public class ArbitraryDataStorageManager extends Thread { } ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - // Make sure to exclude metadata-only resources if requested - if (!includeMetadataOnly) { - if (arbitraryTransactionData.getMetadataHash() != null) { - if (contents.length == 1) { - String metadataHash58 = Base58.encode(arbitraryTransactionData.getMetadataHash()); - if (Objects.equals(metadataHash58, contents[0])) { - // We only have the metadata file for this resource, not the actual data, so exclude it - continue; - } + // Make sure to exclude metadata-only resources + if (arbitraryTransactionData.getMetadataHash() != null) { + if (contents.length == 1) { + String metadataHash58 = Base58.encode(arbitraryTransactionData.getMetadataHash()); + if (Objects.equals(metadataHash58, contents[0])) { + // We only have the metadata file for this resource, not the actual data, so exclude it + continue; } } } @@ -311,10 +307,69 @@ public class ArbitraryDataStorageManager extends Thread { // Sort by newest first arbitraryTransactionDataList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed()); - // Update cache - this.hostedTransactions = arbitraryTransactionDataList; + return arbitraryTransactionDataList; + } + // Hosted data - return ArbitraryTransactionUtils.limitOffsetTransactions(arbitraryTransactionDataList, limit, offset); + public List listAllHostedTransactions(Repository repository, Integer limit, Integer offset) { + // Load from cache if we can, to avoid disk reads + + if (this.hostedTransactions != null) { + return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset); + } + + this.hostedTransactions = this.loadAllHostedTransactions(repository); + + return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset); + } + + /** + * searchHostedTransactions + * Allow to run a query against hosted data names and return matches if there are any + * @param repository + * @param query + * @param limit + * @param offset + * @return + */ + + public List searchHostedTransactions(Repository repository, String query, Integer limit, Integer offset) { + // Load from results cache if we can (results that exists for the same query), to avoid disk reads + if (this.searchResultsTransactions != null && this.searchQuery.equals(query.toLowerCase())) { + return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset); + } + + // Using cache if we can, to avoid disk reads + if (this.hostedTransactions == null) { + this.hostedTransactions = this.loadAllHostedTransactions(repository); + } + + this.searchQuery = query.toLowerCase(); //set the searchQuery so that it can be checked on the next call + + List searchResultsList = new ArrayList<>(); + + // Loop through cached hostedTransactions + for (ArbitraryTransactionData atd : this.hostedTransactions) { + try { + if (atd.getName() != null && atd.getName().toLowerCase().contains(this.searchQuery)) { + searchResultsList.add(atd); + } + else if (atd.getIdentifier() != null && atd.getIdentifier().toLowerCase().contains(this.searchQuery)) { + searchResultsList.add(atd); + } + + } catch (Exception e) { + continue; + } + } + + // Sort by newest first + searchResultsList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed()); + + // Update cache + this.searchResultsTransactions = searchResultsList; + + return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset); } /** @@ -338,7 +393,7 @@ public class ArbitraryDataStorageManager extends Thread { && path.getFileName().toString().length() > 32) .collect(Collectors.toList()); } - catch (IOException e) { + catch (IOException | UncheckedIOException e) { LOGGER.info("Unable to walk through hosted data: {}", e.getMessage()); } @@ -467,7 +522,7 @@ public class ArbitraryDataStorageManager extends Thread { long maxStoragePerName = this.storageCapacityPerName(threshold); // Fetch all hosted transactions - List hostedTransactions = this.listAllHostedTransactions(repository, null, null, true); + List hostedTransactions = this.listAllHostedTransactions(repository, null, null); for (ArbitraryTransactionData transactionData : hostedTransactions) { String transactionName = transactionData.getName(); if (!Objects.equals(name, transactionName)) { diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index 2f9c3121..6996acbb 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -16,6 +16,7 @@ import org.bitcoinj.core.ECKey; import org.qortal.account.PrivateKeyAccount; import org.qortal.api.model.crosschain.TradeBotCreateRequest; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult; import org.qortal.crosschain.*; import org.qortal.data.at.ATData; @@ -213,7 +214,7 @@ public class TradeBot implements Listener { @Override public void listen(Event event) { - if (!(event instanceof Controller.NewBlockEvent)) + if (!(event instanceof Synchronizer.NewChainTipEvent)) return; synchronized (this) { diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index a5a1ab12..f66ea939 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -58,9 +58,14 @@ public abstract class Bitcoiny implements ForeignBlockchain { * i.e. keys with transactions but with no unspent outputs. */ protected final Set spentKeys = Collections.synchronizedSet(new HashSet<>()); - /** How many bitcoinj wallet keys to generate in each batch. */ + /** How many wallet keys to generate in each batch. */ private static final int WALLET_KEY_LOOKAHEAD_INCREMENT = 3; + /** How many wallet keys to generate when using bitcoinj as the data provider. + * We must use a higher value here since we are unable to request multiple batches of keys. + * Without this, the bitcoinj state can be missing transactions, causing errors such as "insufficient balance". */ + private static final int WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ = 50; + /** Byte offset into raw block headers to block timestamp. */ private static final int TIMESTAMP_OFFSET = 4 + 32 + 32; @@ -99,8 +104,9 @@ public abstract class Bitcoiny implements ForeignBlockchain { try { ScriptType addressType = Address.fromString(this.params, address).getOutputScriptType(); - return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH; + return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH || addressType == ScriptType.P2WPKH; } catch (AddressFormatException e) { + LOGGER.error(String.format("Unrecognised address format: %s", address)); return false; } } @@ -404,7 +410,7 @@ public abstract class Bitcoiny implements ForeignBlockchain { Set keySet = new HashSet<>(); // Set the number of consecutive empty batches required before giving up - final int numberOfAdditionalBatchesToSearch = 5; + final int numberOfAdditionalBatchesToSearch = 7; int unusedCounter = 0; int ki = 0; @@ -470,6 +476,9 @@ public abstract class Bitcoiny implements ForeignBlockchain { List inputs = new ArrayList<>(); List outputs = new ArrayList<>(); + boolean anyOutputAddressInWallet = false; + boolean transactionInvolvesExternalWallet = false; + for (BitcoinyTransaction.Input input : t.inputs) { try { BitcoinyTransaction t2 = getTransaction(input.outputTxHash); @@ -483,6 +492,9 @@ public abstract class Bitcoiny implements ForeignBlockchain { total += inputAmount; addressInWallet = true; } + else { + transactionInvolvesExternalWallet = true; + } inputs.add(new SimpleTransaction.Input(sender, inputAmount, addressInWallet)); } } @@ -496,12 +508,16 @@ public abstract class Bitcoiny implements ForeignBlockchain { for (String address : output.addresses) { boolean addressInWallet = false; if (keySet.contains(address)) { - if (total > 0L) { + if (total > 0L) { // Change returned from sent amount amount -= (total - output.value); - } else { + } else { // Amount received amount += output.value; } addressInWallet = true; + anyOutputAddressInWallet = true; + } + else { + transactionInvolvesExternalWallet = true; } outputs.add(new SimpleTransaction.Output(address, output.value, addressInWallet)); } @@ -510,6 +526,17 @@ public abstract class Bitcoiny implements ForeignBlockchain { } } long fee = totalInputAmount - totalOutputAmount; + + if (!anyOutputAddressInWallet) { + // No outputs relate to this wallet - check if any inputs did (which is signified by a positive total) + if (total > 0) { + amount = total * -1; + } + } + else if (!transactionInvolvesExternalWallet) { + // All inputs and outputs relate to this wallet, so the balance should be unaffected + amount = 0; + } return new SimpleTransaction(t.txHash, t.timestamp, amount, fee, inputs, outputs); } @@ -602,7 +629,7 @@ public abstract class Bitcoiny implements ForeignBlockchain { this.keyChain = this.wallet.getActiveKeyChain(); // Set up wallet's key chain - this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ); this.keyChain.maybeLookAhead(); } diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index 7f1eb4c4..6d6cfb15 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -401,13 +401,36 @@ public class ElectrumX extends BitcoinyBlockchainProvider { String scriptPubKey = (String) ((JSONObject) outputJson.get("scriptPubKey")).get("hex"); long value = BigDecimal.valueOf((Double) outputJson.get("value")).setScale(8).unscaledValue().longValue(); - // address too, if present + // address too, if present in the "addresses" array List addresses = null; Object addressesObj = ((JSONObject) outputJson.get("scriptPubKey")).get("addresses"); if (addressesObj instanceof JSONArray) { addresses = new ArrayList<>(); - for (Object addressObj : (JSONArray) addressesObj) + for (Object addressObj : (JSONArray) addressesObj) { addresses.add((String) addressObj); + } + } + + // some peers return a single "address" string + Object addressObj = ((JSONObject) outputJson.get("scriptPubKey")).get("address"); + if (addressObj instanceof String) { + if (addresses == null) { + addresses = new ArrayList<>(); + } + addresses.add((String) addressObj); + } + + // For the purposes of Qortal we require all outputs to contain addresses + // Some servers omit this info, causing problems down the line with balance calculations + // Update: it turns out that they were just using a different key - "address" instead of "addresses" + // The code below can remain in place, just in case a peer returns a missing address in the future + if (addresses == null || addresses.isEmpty()) { + if (this.currentServer != null) { + this.uselessServers.add(this.currentServer); + this.closeServer(this.currentServer); + } + LOGGER.info("No output addresses returned for transaction {}", txHash); + throw new ForeignBlockchainException(String.format("No output addresses returned for transaction %s", txHash)); } outputs.add(new BitcoinyTransaction.Output(scriptPubKey, value, addresses)); diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java index 94f41d18..17c1acac 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java @@ -9,12 +9,16 @@ public class ArbitraryRelayInfo { private final String signature58; private final Peer peer; private final Long timestamp; + private final Long requestTime; + private final Integer requestHops; - public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp) { + public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp, Long requestTime, Integer requestHops) { this.hash58 = hash58; this.signature58 = signature58; this.peer = peer; this.timestamp = timestamp; + this.requestTime = requestTime; + this.requestHops = requestHops; } public boolean isValid() { @@ -38,6 +42,14 @@ public class ArbitraryRelayInfo { return timestamp; } + public Long getRequestTime() { + return this.requestTime; + } + + public Integer getRequestHops() { + return this.requestHops; + } + @Override public String toString() { return String.format("%s = %s, %s, %d", this.hash58, this.signature58, this.peer, this.timestamp); diff --git a/src/main/java/org/qortal/network/Handshake.java b/src/main/java/org/qortal/network/Handshake.java index d88654cf..cdcff1d7 100644 --- a/src/main/java/org/qortal/network/Handshake.java +++ b/src/main/java/org/qortal/network/Handshake.java @@ -74,6 +74,12 @@ public enum Handshake { peer.setPeersConnectionTimestamp(peersConnectionTimestamp); peer.setPeersVersion(versionString, version); + // Ensure the peer is running at least the version specified in MIN_PEER_VERSION + if (peer.isAtLeastVersion(MIN_PEER_VERSION) == false) { + LOGGER.debug(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString)); + return null; + } + if (Settings.getInstance().getAllowConnectionsWithOlderPeerVersions() == false) { // Ensure the peer is running at least the minimum version allowed for connections final String minPeerVersion = Settings.getInstance().getMinPeerVersion(); @@ -258,6 +264,9 @@ public enum Handshake { private static final long PEER_VERSION_131 = 0x0100030001L; + /** Minimum peer version that we are allowed to communicate with */ + private static final String MIN_PEER_VERSION = "3.1.0"; + private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes private static final int POW_DIFFICULTY_PRE_131 = 8; // leading zero bits // Can always be made harder in the future... diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 2a25864a..5de0b84d 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -1,5 +1,6 @@ package org.qortal.network; +import com.dosse.upnp.UPnP; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters; @@ -7,7 +8,6 @@ import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters; import org.qortal.block.BlockChain; import org.qortal.controller.Controller; import org.qortal.controller.arbitrary.ArbitraryDataFileListManager; -import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.crypto.Crypto; import org.qortal.data.block.BlockData; import org.qortal.data.network.PeerData; @@ -183,6 +183,14 @@ public class Network { } } + // Attempt to set up UPnP. All errors are ignored. + if (Settings.getInstance().isUPnPEnabled()) { + UPnP.openPortTCP(Settings.getInstance().getListenPort()); + } + else { + UPnP.closePortTCP(Settings.getInstance().getListenPort()); + } + // Start up first networking thread networkEPC.start(); } @@ -243,12 +251,15 @@ public class Network { public boolean requestDataFromPeer(String peerAddressString, byte[] signature) { if (peerAddressString != null) { PeerAddress peerAddress = PeerAddress.fromString(peerAddressString); + PeerData peerData = null; // Reuse an existing PeerData instance if it's already in the known peers list - PeerData peerData = this.allKnownPeers.stream() - .filter(knownPeerData -> knownPeerData.getAddress().equals(peerAddress)) - .findFirst() - .orElse(null); + synchronized (this.allKnownPeers) { + peerData = this.allKnownPeers.stream() + .filter(knownPeerData -> knownPeerData.getAddress().equals(peerAddress)) + .findFirst() + .orElse(null); + } if (peerData == null) { // Not a known peer, so we need to create one @@ -263,10 +274,13 @@ public class Network { } // Check if we're already connected to and handshaked with this peer - Peer connectedPeer = this.connectedPeers.stream() - .filter(p -> p.getPeerData().getAddress().equals(peerAddress)) - .findFirst() - .orElse(null); + Peer connectedPeer = null; + synchronized (this.connectedPeers) { + connectedPeer = this.connectedPeers.stream() + .filter(p -> p.getPeerData().getAddress().equals(peerAddress)) + .findFirst() + .orElse(null); + } boolean isConnected = (connectedPeer != null); boolean isHandshaked = this.getHandshakedPeers().stream() @@ -1178,7 +1192,12 @@ public class Network { public void onExternalIpUpdate(String ipAddress) { LOGGER.info("External IP address updated to {}", ipAddress); - ArbitraryDataManager.getInstance().broadcastHostedSignatureList(); + //ArbitraryDataManager.getInstance().broadcastHostedSignatureList(); + } + + public String getOurExternalIpAddress() { + // FUTURE: replace port if UPnP is active, as it will be more accurate + return this.ourExternalIpAddress; } diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java index 008b3edd..32ba3fa7 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java @@ -1,6 +1,8 @@ package org.qortal.network.message; import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.qortal.data.network.PeerData; import org.qortal.transform.TransformationException; import org.qortal.transform.Transformer; import org.qortal.utils.Serialization; @@ -16,22 +18,38 @@ public class ArbitraryDataFileListMessage extends Message { private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; private static final int HASH_LENGTH = Transformer.SHA256_LENGTH; + private static final int MAX_PEER_ADDRESS_LENGTH = PeerData.MAX_PEER_ADDRESS_SIZE; private final byte[] signature; private final List hashes; + private Long requestTime; + private Integer requestHops; + private String peerAddress; + private Boolean isRelayPossible; - public ArbitraryDataFileListMessage(byte[] signature, List hashes) { + + public ArbitraryDataFileListMessage(byte[] signature, List hashes, Long requestTime, + Integer requestHops, String peerAddress, boolean isRelayPossible) { super(MessageType.ARBITRARY_DATA_FILE_LIST); this.signature = signature; this.hashes = hashes; + this.requestTime = requestTime; + this.requestHops = requestHops; + this.peerAddress = peerAddress; + this.isRelayPossible = isRelayPossible; } - public ArbitraryDataFileListMessage(int id, byte[] signature, List hashes) { + public ArbitraryDataFileListMessage(int id, byte[] signature, List hashes, Long requestTime, + Integer requestHops, String peerAddress, boolean isRelayPossible) { super(id, MessageType.ARBITRARY_DATA_FILE_LIST); this.signature = signature; this.hashes = hashes; + this.requestTime = requestTime; + this.requestHops = requestHops; + this.peerAddress = peerAddress; + this.isRelayPossible = isRelayPossible; } public List getHashes() { @@ -48,9 +66,6 @@ public class ArbitraryDataFileListMessage extends Message { int count = bytes.getInt(); - if (bytes.remaining() != count * HASH_LENGTH) - return null; - List hashes = new ArrayList<>(); for (int i = 0; i < count; ++i) { @@ -59,7 +74,26 @@ public class ArbitraryDataFileListMessage extends Message { hashes.add(hash); } - return new ArbitraryDataFileListMessage(id, signature, hashes); + Long requestTime = null; + Integer requestHops = null; + String peerAddress = null; + boolean isRelayPossible = true; // Legacy versions only send this message when relaying is possible + + // The remaining fields are optional + + if (bytes.hasRemaining()) { + + requestTime = bytes.getLong(); + + requestHops = bytes.getInt(); + + peerAddress = Serialization.deserializeSizedStringV2(bytes, MAX_PEER_ADDRESS_LENGTH); + + isRelayPossible = bytes.getInt() > 0; + + } + + return new ArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops, peerAddress, isRelayPossible); } @Override @@ -75,6 +109,20 @@ public class ArbitraryDataFileListMessage extends Message { bytes.write(hash); } + if (this.requestTime == null) { // To maintain backwards support + return bytes.toByteArray(); + } + + // The remaining fields are optional + + bytes.write(Longs.toByteArray(this.requestTime)); + + bytes.write(Ints.toByteArray(this.requestHops)); + + Serialization.serializeSizedStringV2(bytes, this.peerAddress); + + bytes.write(Ints.toByteArray(this.isRelayPossible ? 1 : 0)); + return bytes.toByteArray(); } catch (IOException e) { return null; @@ -82,9 +130,49 @@ public class ArbitraryDataFileListMessage extends Message { } public ArbitraryDataFileListMessage cloneWithNewId(int newId) { - ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes); + ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes, + this.requestTime, this.requestHops, this.peerAddress, this.isRelayPossible); clone.setId(newId); return clone; } + public void removeOptionalStats() { + this.requestTime = null; + this.requestHops = null; + this.peerAddress = null; + this.isRelayPossible = null; + } + + public Long getRequestTime() { + return this.requestTime; + } + + public void setRequestTime(Long requestTime) { + this.requestTime = requestTime; + } + + public Integer getRequestHops() { + return this.requestHops; + } + + public void setRequestHops(Integer requestHops) { + this.requestHops = requestHops; + } + + public String getPeerAddress() { + return this.peerAddress; + } + + public void setPeerAddress(String peerAddress) { + this.peerAddress = peerAddress; + } + + public Boolean isRelayPossible() { + return this.isRelayPossible; + } + + public void setIsRelayPossible(Boolean isRelayPossible) { + this.isRelayPossible = isRelayPossible; + } + } diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java index d87e9685..b9f24e29 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java @@ -1,6 +1,8 @@ package org.qortal.network.message; import com.google.common.primitives.Ints; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.repository.DataException; import org.qortal.transform.Transformer; @@ -12,6 +14,8 @@ import java.nio.ByteBuffer; public class ArbitraryDataFileMessage extends Message { + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileMessage.class); + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; private final byte[] signature; @@ -52,6 +56,7 @@ public class ArbitraryDataFileMessage extends Message { return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile); } catch (DataException e) { + LOGGER.info("Unable to process received file: {}", e.getMessage()); return null; } } diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java new file mode 100644 index 00000000..709f9782 --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java @@ -0,0 +1,117 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.qortal.data.network.OnlineAccountData; +import org.qortal.transform.Transformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * For requesting online accounts info from remote peer, given our list of online accounts. + * + * Different format to V1: + * V1 is: number of entries, then timestamp + pubkey for each entry + * V2 is: groups of: number of entries, timestamp, then pubkey for each entry + * + * Also V2 only builds online accounts message once! + */ +public class GetOnlineAccountsV2Message extends Message { + private List onlineAccounts; + private byte[] cachedData; + + public GetOnlineAccountsV2Message(List onlineAccounts) { + this(-1, onlineAccounts); + } + + private GetOnlineAccountsV2Message(int id, List onlineAccounts) { + super(id, MessageType.GET_ONLINE_ACCOUNTS_V2); + + this.onlineAccounts = onlineAccounts; + } + + public List getOnlineAccounts() { + return this.onlineAccounts; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + int accountCount = bytes.getInt(); + + List onlineAccounts = new ArrayList<>(accountCount); + + while (accountCount > 0) { + long timestamp = bytes.getLong(); + + for (int i = 0; i < accountCount; ++i) { + byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + bytes.get(publicKey); + + onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey)); + } + + if (bytes.hasRemaining()) { + accountCount = bytes.getInt(); + } else { + // we've finished + accountCount = 0; + } + } + + return new GetOnlineAccountsV2Message(id, onlineAccounts); + } + + @Override + protected synchronized byte[] toData() { + if (this.cachedData != null) + return this.cachedData; + + // Shortcut in case we have no online accounts + if (this.onlineAccounts.isEmpty()) { + this.cachedData = Ints.toByteArray(0); + return this.cachedData; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (int i = 0; i < this.onlineAccounts.size(); ++i) { + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + Long timestamp = onlineAccountData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + this.onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH; + + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (int i = 0; i < this.onlineAccounts.size(); ++i) { + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + + if (onlineAccountData.getTimestamp() == timestamp) + bytes.write(onlineAccountData.getPublicKey()); + } + } + + this.cachedData = bytes.toByteArray(); + return this.cachedData; + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/network/message/Message.java b/src/main/java/org/qortal/network/message/Message.java index ad6b7ba8..5d95ef48 100644 --- a/src/main/java/org/qortal/network/message/Message.java +++ b/src/main/java/org/qortal/network/message/Message.java @@ -78,6 +78,8 @@ public abstract class Message { ONLINE_ACCOUNTS(80), GET_ONLINE_ACCOUNTS(81), + ONLINE_ACCOUNTS_V2(82), + GET_ONLINE_ACCOUNTS_V2(83), ARBITRARY_DATA(90), GET_ARBITRARY_DATA(91), diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java new file mode 100644 index 00000000..f0fce81e --- /dev/null +++ b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java @@ -0,0 +1,124 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.qortal.data.network.OnlineAccountData; +import org.qortal.transform.Transformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * For sending online accounts info to remote peer. + * + * Different format to V1: + * V1 is: number of entries, then timestamp + sig + pubkey for each entry + * V2 is: groups of: number of entries, timestamp, then sig + pubkey for each entry + * + * Also V2 only builds online accounts message once! + */ +public class OnlineAccountsV2Message extends Message { + private List onlineAccounts; + private byte[] cachedData; + + public OnlineAccountsV2Message(List onlineAccounts) { + this(-1, onlineAccounts); + } + + private OnlineAccountsV2Message(int id, List onlineAccounts) { + super(id, MessageType.ONLINE_ACCOUNTS_V2); + + this.onlineAccounts = onlineAccounts; + } + + public List getOnlineAccounts() { + return this.onlineAccounts; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + int accountCount = bytes.getInt(); + + List onlineAccounts = new ArrayList<>(accountCount); + + while (accountCount > 0) { + long timestamp = bytes.getLong(); + + for (int i = 0; i < accountCount; ++i) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; + bytes.get(signature); + + byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + bytes.get(publicKey); + + onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey)); + } + + if (bytes.hasRemaining()) { + accountCount = bytes.getInt(); + } else { + // we've finished + accountCount = 0; + } + } + + return new OnlineAccountsV2Message(id, onlineAccounts); + } + + @Override + protected synchronized byte[] toData() { + if (this.cachedData != null) + return this.cachedData; + + // Shortcut in case we have no online accounts + if (this.onlineAccounts.isEmpty()) { + this.cachedData = Ints.toByteArray(0); + return this.cachedData; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (int i = 0; i < this.onlineAccounts.size(); ++i) { + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + Long timestamp = onlineAccountData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + this.onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH); + + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (int i = 0; i < this.onlineAccounts.size(); ++i) { + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + + if (onlineAccountData.getTimestamp() == timestamp) { + bytes.write(onlineAccountData.getSignature()); + + bytes.write(onlineAccountData.getPublicKey()); + } + } + } + + this.cachedData = bytes.toByteArray(); + return this.cachedData; + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 45f89697..779c29f5 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -181,6 +181,8 @@ public class Settings { private boolean isTestNet = false; /** Port number for inbound peer-to-peer connections. */ private Integer listenPort; + /** Whether to attempt to open the listen port via UPnP */ + private boolean uPnPEnabled = true; /** Minimum number of peers to allow block minting / synchronization. */ private int minBlockchainPeers = 5; /** Target number of outbound connections to peers we should make. */ @@ -195,7 +197,7 @@ public class Settings { private int maxRetries = 2; /** Minimum peer version number required in order to sync with them */ - private String minPeerVersion = "3.0.1"; + private String minPeerVersion = "3.1.0"; /** Whether to allow connections with peers below minPeerVersion * If true, we won't sync with them but they can still sync with us, and will show in the peers list * If false, sync will be blocked both ways, and they will not appear in the peers list */ @@ -629,6 +631,10 @@ public class Settings { return this.bindAddress; } + public boolean isUPnPEnabled() { + return this.uPnPEnabled; + } + public int getMinBlockchainPeers() { return this.minBlockchainPeers; } diff --git a/src/main/java/org/qortal/transaction/TransferPrivsTransaction.java b/src/main/java/org/qortal/transaction/TransferPrivsTransaction.java index f77dac15..f6a9de68 100644 --- a/src/main/java/org/qortal/transaction/TransferPrivsTransaction.java +++ b/src/main/java/org/qortal/transaction/TransferPrivsTransaction.java @@ -58,7 +58,9 @@ public class TransferPrivsTransaction extends Transaction { return ValidationResult.INVALID_ADDRESS; // Check recipient is new account - if (this.repository.getAccountRepository().accountExists(this.transferPrivsTransactionData.getRecipient())) + AccountData recipientAccountData = this.repository.getAccountRepository().getAccount(this.transferPrivsTransactionData.getRecipient()); + // Non-existent account data is OK, but if account data exists then reference needs to be null + if (recipientAccountData != null && recipientAccountData.getReference() != null) return ValidationResult.ACCOUNT_ALREADY_EXISTS; // Check sender has funds for fee diff --git a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java new file mode 100644 index 00000000..b1c5ec4f --- /dev/null +++ b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java @@ -0,0 +1,114 @@ +package org.qortal.test.network; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.junit.Test; +import org.qortal.data.network.OnlineAccountData; +import org.qortal.network.message.*; +import org.qortal.transform.Transformer; + +import java.nio.ByteBuffer; +import java.security.Security; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class OnlineAccountsTests { + + private static final Random RANDOM = new Random(); + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + } + + + @Test + public void testGetOnlineAccountsV2() throws Message.MessageException { + List onlineAccountsOut = generateOnlineAccounts(false); + + Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut); + + byte[] messageBytes = messageOut.toBytes(); + ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes); + + GetOnlineAccountsV2Message messageIn = (GetOnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer); + + List onlineAccountsIn = messageIn.getOnlineAccounts(); + + assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size()); + assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut)); + + Message oldMessageOut = new GetOnlineAccountsMessage(onlineAccountsOut); + byte[] oldMessageBytes = oldMessageOut.toBytes(); + + long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count(); + + System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d", + onlineAccountsOut.size(), + numTimestamps, + numTimestamps != 1 ? "s" : "", + oldMessageBytes.length, + messageBytes.length)); + } + + @Test + public void testOnlineAccountsV2() throws Message.MessageException { + List onlineAccountsOut = generateOnlineAccounts(true); + + Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut); + + byte[] messageBytes = messageOut.toBytes(); + ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes); + + OnlineAccountsV2Message messageIn = (OnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer); + + List onlineAccountsIn = messageIn.getOnlineAccounts(); + + assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size()); + assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut)); + + Message oldMessageOut = new OnlineAccountsMessage(onlineAccountsOut); + byte[] oldMessageBytes = oldMessageOut.toBytes(); + + long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count(); + + System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d", + onlineAccountsOut.size(), + numTimestamps, + numTimestamps != 1 ? "s" : "", + oldMessageBytes.length, + messageBytes.length)); + } + + private List generateOnlineAccounts(boolean withSignatures) { + List onlineAccounts = new ArrayList<>(); + + int numTimestamps = RANDOM.nextInt(2) + 1; // 1 or 2 + + for (int t = 0; t < numTimestamps; ++t) { + int numAccounts = RANDOM.nextInt(3000); + + for (int a = 0; a < numAccounts; ++a) { + byte[] sig = null; + if (withSignatures) { + sig = new byte[Transformer.SIGNATURE_LENGTH]; + RANDOM.nextBytes(sig); + } + + byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + RANDOM.nextBytes(pubkey); + + onlineAccounts.add(new OnlineAccountData(t << 32, sig, pubkey)); + } + } + + return onlineAccounts; + } + +}