diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..527db761 Binary files /dev/null and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index 88056ba0..db0a997c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ /settings*.json /testchain.json /run-testnet.sh +/.idea +/qortal.iml +*.DS_Store diff --git a/AutoUpdates.md b/AutoUpdates.md index e565fb06..7f248246 100644 --- a/AutoUpdates.md +++ b/AutoUpdates.md @@ -2,6 +2,7 @@ ## TL;DR: how-to +* Prepare new release version (see way below for details) * Assuming you are in git 'master' branch, at HEAD * Shutdown local node if running * Build auto-update download: `tools/build-auto-update.sh` - uploads auto-update file into new git branch @@ -59,4 +60,12 @@ $ java -cp qortal.jar org.qortal.XorUpdate usage: XorUpdate $ java -cp qortal.jar org.qortal.XorUpdate qortal.jar qortal.update $ -``` \ No newline at end of file +``` + +## Preparing new release version + +* Shutdown local node +* Modify `pom.xml` and increase version inside `` tag +* Commit new `pom.xml` and push to github, e.g. `git commit -m 'Bumped to v1.4.2' -- pom.xml; git push` +* Tag this new commit with same version: `git tag v1.4.2` +* Push tag up to github: `git push origin v1.4.2` diff --git a/WindowsInstaller/Install Files/qortal.jar b/WindowsInstaller/Install Files/qortal.jar index 167c4f12..230dda80 100755 Binary files a/WindowsInstaller/Install Files/qortal.jar and b/WindowsInstaller/Install Files/qortal.jar differ diff --git a/WindowsInstaller/Qortal.aip b/WindowsInstaller/Qortal.aip index cc878c79..e19f3664 100755 --- a/WindowsInstaller/Qortal.aip +++ b/WindowsInstaller/Qortal.aip @@ -19,10 +19,10 @@ - + - + @@ -174,7 +174,7 @@ - + diff --git a/pom.xml b/pom.xml index cb155d25..6697cc81 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 1.4.1 + 1.5.0 jar true diff --git a/src/.DS_Store b/src/.DS_Store new file mode 100644 index 00000000..0c7ee143 Binary files /dev/null and b/src/.DS_Store differ diff --git a/src/main/.DS_Store b/src/main/.DS_Store new file mode 100644 index 00000000..b5507e38 Binary files /dev/null and b/src/main/.DS_Store differ diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index 81eb0bf8..c295b90b 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -547,7 +547,7 @@ public class AdminResource { blockchainLock.lockInterruptibly(); try { - repository.exportNodeLocalData(); + repository.exportNodeLocalData(true); return "true"; } finally { blockchainLock.unlock(); @@ -628,25 +628,9 @@ public class AdminResource { public String checkpointRepository() { Security.checkApiCallAllowed(request); - try (final Repository repository = RepositoryManager.getRepository()) { - ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); - blockchainLock.lockInterruptibly(); - - try { - repository.checkpoint(true); - repository.saveChanges(); - - return "true"; - } finally { - blockchainLock.unlock(); - } - } catch (InterruptedException e) { - // We couldn't lock blockchain to perform checkpoint - return "false"; - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } + return "true"; } @POST diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index ce7d220b..47d82043 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -176,19 +176,26 @@ public class Block { * * @return account-level share "bin" from blockchain config, or null if founder / none found */ - public AccountLevelShareBin getShareBin() { + public AccountLevelShareBin getShareBin(int blockHeight) { if (this.isMinterFounder) return null; final int accountLevel = this.mintingAccountData.getLevel(); if (accountLevel <= 0) - return null; + return null; // level 0 isn't included in any share bins - final AccountLevelShareBin[] shareBinsByLevel = BlockChain.getInstance().getShareBinsByAccountLevel(); + final BlockChain blockChain = BlockChain.getInstance(); + final AccountLevelShareBin[] shareBinsByLevel = blockChain.getShareBinsByAccountLevel(); if (accountLevel > shareBinsByLevel.length) return null; - return shareBinsByLevel[accountLevel]; + if (blockHeight < blockChain.getShareBinFixHeight()) + // Off-by-one bug still in effect + return shareBinsByLevel[accountLevel]; + + // level 1 stored at index 0, level 2 stored at index 1, etc. + return shareBinsByLevel[accountLevel-1]; + } public long distribute(long accountAmount, Map balanceChanges) { @@ -357,7 +364,7 @@ public class Block { System.arraycopy(onlineAccountData.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH); } - byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData.getMinterSignature(), + byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData, minter.getPublicKey(), encodedOnlineAccounts)); // Qortal: minter is always a reward-share, so find actual minter and get their effective minting level @@ -424,7 +431,7 @@ public class Block { int version = this.blockData.getVersion(); byte[] reference = this.blockData.getReference(); - byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData.getMinterSignature(), + byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData, minter.getPublicKey(), this.blockData.getEncodedOnlineAccounts())); // Qortal: minter is always a reward-share, so find actual minter and get their effective minting level @@ -738,11 +745,7 @@ public class Block { if (!(this.minter instanceof PrivateKeyAccount)) throw new IllegalStateException("Block's minter is not a PrivateKeyAccount - can't sign!"); - try { - this.blockData.setMinterSignature(((PrivateKeyAccount) this.minter).sign(BlockTransformer.getBytesForMinterSignature(this.blockData))); - } catch (TransformationException e) { - throw new RuntimeException("Unable to calculate block's minter signature", e); - } + this.blockData.setMinterSignature(((PrivateKeyAccount) this.minter).sign(BlockTransformer.getBytesForMinterSignature(this.blockData))); } /** @@ -1331,6 +1334,9 @@ public class Block { // Give Controller our cached, valid online accounts data (if any) to help reduce CPU load for next block Controller.getInstance().pushLatestBlocksOnlineAccounts(this.cachedValidOnlineAccounts); + + // Log some debugging info relating to the block weight calculation + this.logDebugInfo(); } protected void increaseAccountLevels() throws DataException { @@ -1512,6 +1518,9 @@ public class Block { public void orphan() throws DataException { LOGGER.trace(() -> String.format("Orphaning block %d", this.blockData.getHeight())); + // Log some debugging info relating to the block weight calculation + this.logDebugInfo(); + // Return AT fees and delete AT states from repository orphanAtFeesAndStates(); @@ -1786,7 +1795,7 @@ public class Block { // Find all accounts in share bin. getShareBin() returns null for minter accounts that are also founders, so they are effectively filtered out. AccountLevelShareBin accountLevelShareBin = accountLevelShareBins.get(binIndex); // Object reference compare is OK as all references are read-only from blockchain config. - List binnedAccounts = expandedAccounts.stream().filter(accountInfo -> accountInfo.getShareBin() == accountLevelShareBin).collect(Collectors.toList()); + List binnedAccounts = expandedAccounts.stream().filter(accountInfo -> accountInfo.getShareBin(this.blockData.getHeight()) == accountLevelShareBin).collect(Collectors.toList()); // No online accounts in this bin? Skip to next one if (binnedAccounts.isEmpty()) @@ -1984,4 +1993,33 @@ public class Block { this.repository.getAccountRepository().tidy(); } + private void logDebugInfo() { + try { + if (this.repository == null || this.getMinter() == null || this.getBlockData() == null) + return; + + int minterLevel = Account.getRewardShareEffectiveMintingLevel(this.repository, this.getMinter().getPublicKey()); + + LOGGER.debug(String.format("======= BLOCK %d (%.8s) =======", this.getBlockData().getHeight(), Base58.encode(this.getSignature()))); + LOGGER.debug(String.format("Timestamp: %d", this.getBlockData().getTimestamp())); + LOGGER.debug(String.format("Minter level: %d", minterLevel)); + LOGGER.debug(String.format("Online accounts: %d", this.getBlockData().getOnlineAccountsCount())); + + BlockSummaryData blockSummaryData = new BlockSummaryData(this.getBlockData()); + if (this.getParent() == null || this.getParent().getSignature() == null || blockSummaryData == null) + return; + + blockSummaryData.setMinterLevel(minterLevel); + BigInteger blockWeight = calcBlockWeight(this.getParent().getHeight(), this.getParent().getSignature(), blockSummaryData); + BigInteger keyDistance = calcKeyDistance(this.getParent().getHeight(), this.getParent().getSignature(), blockSummaryData.getMinterPublicKey(), blockSummaryData.getMinterLevel()); + NumberFormat formatter = new DecimalFormat("0.###E0"); + + LOGGER.debug(String.format("Key distance: %s", formatter.format(keyDistance))); + LOGGER.debug(String.format("Weight: %s", formatter.format(blockWeight))); + + } catch (DataException e) { + LOGGER.info(() -> String.format("Unable to log block debugging info: %s", e.getMessage())); + } + } + } diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 43b19468..e6b8db4e 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -71,6 +71,8 @@ public class BlockChain { public enum FeatureTrigger { atFindNextTransactionFix, + newBlockSigHeight, + shareBinFix, calcChainWeightTimestamp; } @@ -377,6 +379,14 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.atFindNextTransactionFix.name()).intValue(); } + public int getNewBlockSigHeight() { + return this.featureTriggers.get(FeatureTrigger.newBlockSigHeight.name()).intValue(); + } + + public int getShareBinFixHeight() { + return this.featureTriggers.get(FeatureTrigger.shareBinFix.name()).intValue(); + } + public long getCalcChainWeightTimestamp() { return this.featureTriggers.get(FeatureTrigger.calcChainWeightTimestamp.name()).longValue(); } diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index f804456f..8b6563f2 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -135,16 +135,19 @@ public class BlockMinter extends Thread { // Disregard peers that have "misbehaved" recently peers.removeIf(Controller.hasMisbehaved); - // Disregard peers that don't have a recent block - peers.removeIf(Controller.hasNoRecentBlock); + // Disregard peers that don't have a recent block, but only if we're not in recovery mode. + // In that mode, we want to allow minting on top of older blocks, to recover stalled networks. + if (Controller.getInstance().getRecoveryMode() == false) + peers.removeIf(Controller.hasNoRecentBlock); // Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from? if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) continue; - // If our latest block isn't recent then we need to synchronize instead of minting. + // If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode. if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp) - continue; + if (Controller.getInstance().getRecoveryMode() == false) + continue; // There are enough peers with a recent block and our latest block is recent // so go ahead and mint a block if possible. @@ -165,6 +168,14 @@ public class BlockMinter extends Thread { // Do we need to build any potential new blocks? List newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList()); + // We might need to sit the next block out, if one of our minting accounts signed the previous one + final byte[] previousBlockMinter = previousBlockData.getMinterPublicKey(); + final boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter)); + if (mintedLastBlock) { + LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one")); + continue; + } + for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) { // First block does the AT heavy-lifting if (newBlocks.isEmpty()) { @@ -282,15 +293,17 @@ public class BlockMinter extends Thread { RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey()); if (rewardShareData != null) { - LOGGER.info(String.format("Minted block %d, sig %.8s by %s on behalf of %s", + LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s", newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature()), + Base58.encode(newBlock.getParent().getSignature()), rewardShareData.getMinter(), rewardShareData.getRecipient())); } else { - LOGGER.info(String.format("Minted block %d, sig %.8s by %s", + LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s", newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature()), + Base58.encode(newBlock.getParent().getSignature()), newBlock.getMinter().getAddress())); } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 08a1e6a8..a7d028bc 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -67,8 +67,8 @@ import org.qortal.gui.SysTray; import org.qortal.network.Network; import org.qortal.network.Peer; import org.qortal.network.message.ArbitraryDataMessage; -import org.qortal.network.message.BlockMessage; import org.qortal.network.message.BlockSummariesMessage; +import org.qortal.network.message.CachedBlockMessage; import org.qortal.network.message.GetArbitraryDataMessage; import org.qortal.network.message.GetBlockMessage; import org.qortal.network.message.GetBlockSummariesMessage; @@ -121,6 +121,7 @@ public class Controller extends Thread { private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms + private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms // To do with online accounts list private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms @@ -143,16 +144,15 @@ public class Controller extends Thread { private ExecutorService callbackExecutor = Executors.newFixedThreadPool(3); private volatile boolean notifyGroupMembershipChange = false; - private static final int BLOCK_CACHE_SIZE = 10; // To cover typical Synchronizer request + a few spare /** Latest blocks on our chain. Note: tail/last is the latest block. */ private final Deque latestBlocks = new LinkedList<>(); /** Cache of BlockMessages, indexed by block signature */ @SuppressWarnings("serial") - private final LinkedHashMap blockMessageCache = new LinkedHashMap<>() { + private final LinkedHashMap blockMessageCache = new LinkedHashMap<>() { @Override - protected boolean removeEldestEntry(Map.Entry eldest) { - return this.size() > BLOCK_CACHE_SIZE; + protected boolean removeEldestEntry(Map.Entry eldest) { + return this.size() > Settings.getInstance().getBlockCacheSize(); } }; @@ -176,6 +176,11 @@ public class Controller extends Thread { /** Latest block signatures from other peers that we know are on inferior chains. */ List inferiorChainSignatures = new ArrayList<>(); + /** Recovery mode, which is used to bring back a stalled network */ + private boolean recoveryMode = false; + private boolean peersAvailable = true; // peersAvailable must default to true + private long timePeersLastAvailable = 0; + /** * Map of recent requests for ARBITRARY transaction data payloads. *

@@ -319,11 +324,12 @@ public class Controller extends Thread { // Set initial chain height/tip try (final Repository repository = RepositoryManager.getRepository()) { BlockData blockData = repository.getBlockRepository().getLastBlock(); + int blockCacheSize = Settings.getInstance().getBlockCacheSize(); synchronized (this.latestBlocks) { this.latestBlocks.clear(); - for (int i = 0; i < BLOCK_CACHE_SIZE && blockData != null; ++i) { + for (int i = 0; i < blockCacheSize && blockData != null; ++i) { this.latestBlocks.addFirst(blockData); blockData = repository.getBlockRepository().fromHeight(blockData.getHeight() - 1); } @@ -358,6 +364,10 @@ public class Controller extends Thread { } } + public boolean getRecoveryMode() { + return this.recoveryMode; + } + // Entry point public static void main(String[] args) { @@ -536,12 +546,7 @@ public class Controller extends Thread { if (now >= repositoryCheckpointTimestamp + repositoryCheckpointInterval) { repositoryCheckpointTimestamp = now + repositoryCheckpointInterval; - if (Settings.getInstance().getShowCheckpointNotification()) - SysTray.getInstance().showMessage(Translator.INSTANCE.translate("SysTray", "DB_CHECKPOINT"), - Translator.INSTANCE.translate("SysTray", "PERFORMING_DB_CHECKPOINT"), - MessageType.INFO); - - RepositoryManager.checkpoint(true); + RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); } // Give repository a chance to backup (if enabled) @@ -634,6 +639,13 @@ public class Controller extends Thread { // Disregard peers that don't have a recent block peers.removeIf(hasNoRecentBlock); + checkRecoveryModeForPeers(peers); + if (recoveryMode) { + peers = Network.getInstance().getHandshakedPeers(); + peers.removeIf(hasOnlyGenesisBlock); + peers.removeIf(hasMisbehaved); + } + // Check we have enough peers to potentially synchronize if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) return; @@ -644,9 +656,31 @@ public class Controller extends Thread { // Disregard peers that are on the same block as last sync attempt and we didn't like their chain peers.removeIf(hasInferiorChainTip); + final int peersBeforeComparison = peers.size(); + + // Request recent block summaries from the remaining peers, and locate our common block with each + Synchronizer.getInstance().findCommonBlocksWithPeers(peers); + + // Compare the peers against each other, and against our chain, which will return an updated list excluding those without common blocks + peers = Synchronizer.getInstance().comparePeers(peers); + + // We may have added more inferior chain tips when comparing peers, so remove any peers that are currently on those chains + peers.removeIf(hasInferiorChainTip); + + final int peersRemoved = peersBeforeComparison - peers.size(); + if (peersRemoved > 0) + LOGGER.info(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size())); + if (peers.isEmpty()) return; + if (peers.size() > 1) { + StringBuilder finalPeersString = new StringBuilder(); + for (Peer peer : peers) + finalPeersString = finalPeersString.length() > 0 ? finalPeersString.append(", ").append(peer) : finalPeersString.append(peer); + LOGGER.info(String.format("Choosing random peer from: [%s]", finalPeersString.toString())); + } + // Pick random peer to sync with int index = new SecureRandom().nextInt(peers.size()); Peer peer = peers.get(index); @@ -749,6 +783,46 @@ public class Controller extends Thread { } } + private boolean checkRecoveryModeForPeers(List qualifiedPeers) { + List handshakedPeers = Network.getInstance().getHandshakedPeers(); + + if (handshakedPeers.size() > 0) { + // There is at least one handshaked peer + if (qualifiedPeers.isEmpty()) { + // There are no 'qualified' peers - i.e. peers that have a recent block we can sync to + boolean werePeersAvailable = peersAvailable; + peersAvailable = false; + + // If peers only just became unavailable, update our record of the time they were last available + if (werePeersAvailable) + timePeersLastAvailable = NTP.getTime(); + + // If enough time has passed, enter recovery mode, which lifts some restrictions on who we can sync with and when we can mint + if (NTP.getTime() - timePeersLastAvailable > RECOVERY_MODE_TIMEOUT) { + if (recoveryMode == false) { + LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", RECOVERY_MODE_TIMEOUT/60/1000)); + recoveryMode = true; + } + } + } else { + // We now have at least one peer with a recent block, so we can exit recovery mode and sync normally + peersAvailable = true; + if (recoveryMode) { + LOGGER.info("Peers have become available again. Exiting recovery mode..."); + recoveryMode = false; + } + } + } + return recoveryMode; + } + + public void addInferiorChainSignature(byte[] inferiorSignature) { + // Update our list of inferior chain tips + ByteArray inferiorChainSignature = new ByteArray(inferiorSignature); + if (!inferiorChainSignatures.contains(inferiorChainSignature)) + inferiorChainSignatures.add(inferiorChainSignature); + } + public static class StatusChangeEvent implements Event { public StatusChangeEvent() { } @@ -780,7 +854,7 @@ public class Controller extends Thread { actionText = Translator.INSTANCE.translate("SysTray", "MINTING_DISABLED"); } - String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height); + String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height) + "\n" + String.format("Build version: %s", this.buildVersion); SysTray.getInstance().setToolTipText(tooltip); this.callbackExecutor.execute(() -> { @@ -811,7 +885,10 @@ public class Controller extends Thread { repository.saveChanges(); } catch (DataException e) { - LOGGER.error("Repository issue while deleting expired unconfirmed transactions", e); + if (RepositoryManager.isDeadlockRelated(e)) + LOGGER.info("Couldn't delete some expired, unconfirmed transactions this round"); + else + LOGGER.error("Repository issue while deleting expired unconfirmed transactions", e); } } @@ -935,6 +1012,7 @@ public class Controller extends Thread { public void onNewBlock(BlockData latestBlockData) { // Protective copy BlockData blockDataCopy = new BlockData(latestBlockData); + int blockCacheSize = Settings.getInstance().getBlockCacheSize(); synchronized (this.latestBlocks) { BlockData cachedChainTip = this.latestBlocks.peekLast(); @@ -944,7 +1022,7 @@ public class Controller extends Thread { this.latestBlocks.addLast(latestBlockData); // Trim if necessary - if (this.latestBlocks.size() >= BLOCK_CACHE_SIZE) + if (this.latestBlocks.size() >= blockCacheSize) this.latestBlocks.pollFirst(); } else { if (cachedChainTip != null) @@ -1152,14 +1230,15 @@ public class Controller extends Thread { ByteArray signatureAsByteArray = new ByteArray(signature); - BlockMessage cachedBlockMessage = this.blockMessageCache.get(signatureAsByteArray); + CachedBlockMessage cachedBlockMessage = this.blockMessageCache.get(signatureAsByteArray); + int blockCacheSize = Settings.getInstance().getBlockCacheSize(); // Check cached latest block message if (cachedBlockMessage != null) { this.stats.getBlockMessageStats.cacheHits.incrementAndGet(); // We need to duplicate it to prevent multiple threads setting ID on the same message - BlockMessage clonedBlockMessage = cachedBlockMessage.cloneWithNewId(message.getId()); + CachedBlockMessage clonedBlockMessage = cachedBlockMessage.cloneWithNewId(message.getId()); if (!peer.sendMessage(clonedBlockMessage)) peer.disconnect("failed to send block"); @@ -1187,15 +1266,18 @@ public class Controller extends Thread { Block block = new Block(repository, blockData); - BlockMessage blockMessage = new BlockMessage(block); + CachedBlockMessage blockMessage = new CachedBlockMessage(block); blockMessage.setId(message.getId()); // This call also causes the other needed data to be pulled in from repository - if (!peer.sendMessage(blockMessage)) + if (!peer.sendMessage(blockMessage)) { peer.disconnect("failed to send block"); + // Don't fall-through to caching because failure to send might be from failure to build message + return; + } // If request is for a recent block, cache it - if (getChainHeight() - blockData.getHeight() <= BLOCK_CACHE_SIZE) { + if (getChainHeight() - blockData.getHeight() <= blockCacheSize) { this.stats.getBlockMessageStats.cacheFills.incrementAndGet(); this.blockMessageCache.put(new ByteArray(blockData.getSignature()), blockMessage); @@ -1209,6 +1291,18 @@ public class Controller extends Thread { TransactionMessage transactionMessage = (TransactionMessage) message; TransactionData transactionData = transactionMessage.getTransactionData(); + /* + * If we can't obtain blockchain lock immediately, + * e.g. Synchronizer is active, or another transaction is taking a while to validate, + * then we're using up a network thread for ages and clogging things up + * so bail out early + */ + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + if (!blockchainLock.tryLock()) { + LOGGER.trace(() -> String.format("Too busy to import %s transaction %s from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer)); + return; + } + try (final Repository repository = RepositoryManager.getRepository()) { Transaction transaction = Transaction.fromData(repository, transactionData); @@ -1238,6 +1332,8 @@ public class Controller extends Thread { LOGGER.debug(() -> String.format("Imported %s transaction %s from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer)); } catch (DataException e) { LOGGER.error(String.format("Repository issue while processing transaction %s from peer %s", Base58.encode(transactionData.getSignature()), peer), e); + } finally { + blockchainLock.unlock(); } } diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 06850a1b..2d4391b1 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -8,6 +8,7 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; +import java.util.Iterator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -17,6 +18,7 @@ import org.qortal.block.Block; import org.qortal.block.Block.ValidationResult; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; +import org.qortal.data.block.CommonBlockData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.transaction.RewardShareTransactionData; import org.qortal.data.transaction.TransactionData; @@ -39,10 +41,23 @@ public class Synchronizer { private static final Logger LOGGER = LogManager.getLogger(Synchronizer.class); + /** Max number of new blocks we aim to add to chain tip in each sync round */ + private static final int SYNC_BATCH_SIZE = 200; // XXX move to Settings? + + /** Initial jump back of block height when searching for common block with peer */ private static final int INITIAL_BLOCK_STEP = 8; - private static final int MAXIMUM_BLOCK_STEP = 500; + /** Maximum jump back of block height when searching for common block with peer */ + private static final int MAXIMUM_BLOCK_STEP = 128; + + /** Maximum difference in block height between tip and peer's common block before peer is considered TOO DIVERGENT */ private static final int MAXIMUM_COMMON_DELTA = 240; // XXX move to Settings? - private static final int SYNC_BATCH_SIZE = 200; + + /** Maximum number of block signatures we ask from peer in one go */ + private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings? + + /** Number of retry attempts if a peer fails to respond with the requested data */ + private static final int MAXIMUM_RETRIES = 2; // XXX move to Settings? + private static Synchronizer instance; @@ -62,6 +77,377 @@ public class Synchronizer { return instance; } + + /** + * Iterate through a list of supplied peers, and attempt to find our common block with each. + * If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later. + *

+ * Will return SynchronizationResult.OK on success. + *

+ * @param peers + * @return SynchronizationResult.OK if the process completed successfully, or a different SynchronizationResult if something went wrong. + * @throws InterruptedException + */ + public SynchronizationResult findCommonBlocksWithPeers(List peers) throws InterruptedException { + try (final Repository repository = RepositoryManager.getRepository()) { + try { + + if (peers.size() == 0) + return SynchronizationResult.NOTHING_TO_DO; + + // If our latest block is very old, it's best that we don't try and determine the best peers to sync to. + // This is because it can involve very large chain comparisons, which is too intensive. + // In reality, most forking problems occur near the chain tips, so we will reserve this functionality for those situations. + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null) + return SynchronizationResult.REPOSITORY_ISSUE; + + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + LOGGER.debug(String.format("Our latest block is very old, so we won't collect common block info from peers")); + return SynchronizationResult.NOTHING_TO_DO; + } + + LOGGER.debug(String.format("Searching for common blocks with %d peers...", peers.size())); + final long startTime = System.currentTimeMillis(); + int commonBlocksFound = 0; + + for (Peer peer : peers) { + // Are we shutting down? + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + // Check if we can use the cached common block data, by comparing the peer's current chain tip against the peer's chain tip when we last found our common block + if (peer.canUseCachedCommonBlockData()) { + LOGGER.debug(String.format("Skipping peer %s because we already have the latest common block data in our cache. Cached common block sig is %.08s", peer, Base58.encode(peer.getCommonBlockData().getCommonBlockSummary().getSignature()))); + continue; + } + + // Cached data is stale, so clear it and repopulate + peer.setCommonBlockData(null); + + // Search for the common block + Synchronizer.getInstance().findCommonBlockWithPeer(peer, repository); + if (peer.getCommonBlockData() != null) + commonBlocksFound++; + } + + final long totalTimeTaken = System.currentTimeMillis() - startTime; + LOGGER.info(String.format("Finished searching for common blocks with %d peer%s. Found: %d. Total time taken: %d ms", peers.size(), (peers.size() != 1 ? "s" : ""), commonBlocksFound, totalTimeTaken)); + + return SynchronizationResult.OK; + } finally { + repository.discardChanges(); // Free repository locks, if any, also in case anything went wrong + } + } catch (DataException e) { + LOGGER.error("Repository issue during synchronization with peer", e); + return SynchronizationResult.REPOSITORY_ISSUE; + } + } + + /** + * Attempt to find the find our common block with supplied peer. + * If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later. + *

+ * Will return SynchronizationResult.OK on success. + *

+ * @param peer + * @param repository + * @return SynchronizationResult.OK if the process completed successfully, or a different SynchronizationResult if something went wrong. + * @throws InterruptedException + */ + public SynchronizationResult findCommonBlockWithPeer(Peer peer, Repository repository) throws InterruptedException { + try { + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + final int ourInitialHeight = ourLatestBlockData.getHeight(); + + PeerChainTipData peerChainTipData = peer.getChainTipData(); + int peerHeight = peerChainTipData.getLastHeight(); + byte[] peersLastBlockSignature = peerChainTipData.getLastBlockSignature(); + + byte[] ourLastBlockSignature = ourLatestBlockData.getSignature(); + LOGGER.debug(String.format("Fetching summaries from peer %s at height %d, sig %.8s, ts %d; our height %d, sig %.8s, ts %d", peer, + peerHeight, Base58.encode(peersLastBlockSignature), peer.getChainTipData().getLastBlockTimestamp(), + ourInitialHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp())); + + List peerBlockSummaries = new ArrayList<>(); + SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, false, peerBlockSummaries); + if (findCommonBlockResult != SynchronizationResult.OK) { + // Logging performed by fetchSummariesFromCommonBlock() above + peer.setCommonBlockData(null); + return findCommonBlockResult; + } + + // First summary is common block + final BlockData commonBlockData = repository.getBlockRepository().fromSignature(peerBlockSummaries.get(0).getSignature()); + final BlockSummaryData commonBlockSummary = new BlockSummaryData(commonBlockData); + final int commonBlockHeight = commonBlockData.getHeight(); + final byte[] commonBlockSig = commonBlockData.getSignature(); + final String commonBlockSig58 = Base58.encode(commonBlockSig); + LOGGER.debug(String.format("Common block with peer %s is at height %d, sig %.8s, ts %d", peer, + commonBlockHeight, commonBlockSig58, commonBlockData.getTimestamp())); + peerBlockSummaries.remove(0); + + // Store the common block summary against the peer, and the current chain tip (for caching) + peer.setCommonBlockData(new CommonBlockData(commonBlockSummary, peerChainTipData)); + + return SynchronizationResult.OK; + } catch (DataException e) { + LOGGER.error("Repository issue during synchronization with peer", e); + return SynchronizationResult.REPOSITORY_ISSUE; + } + } + + + /** + * Compare a list of peers to determine the best peer(s) to sync to next. + *

+ * Will return a filtered list of peers on success, or an identical list of peers on failure. + * This allows us to fall back to legacy behaviour (random selection from the entire list of peers), if we are unable to make the comparison. + *

+ * @param peers + * @return a list of peers, possibly filtered. + * @throws InterruptedException + */ + public List comparePeers(List peers) throws InterruptedException { + try (final Repository repository = RepositoryManager.getRepository()) { + try { + + // If our latest block is very old, it's best that we don't try and determine the best peers to sync to. + // This is because it can involve very large chain comparisons, which is too intensive. + // In reality, most forking problems occur near the chain tips, so we will reserve this functionality for those situations. + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null) + return peers; + + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + LOGGER.debug(String.format("Our latest block is very old, so we won't filter the peers list")); + return peers; + } + + // Retrieve a list of unique common blocks from this list of peers + List commonBlocks = this.uniqueCommonBlocks(peers); + + // Order common blocks by height, in ascending order + // This is essential for the logic below to make the correct decisions when discarding chains - do not remove + commonBlocks.sort((b1, b2) -> Integer.valueOf(b1.getHeight()).compareTo(Integer.valueOf(b2.getHeight()))); + + // Get our latest height + final int ourHeight = ourLatestBlockData.getHeight(); + + // Create a placeholder to track of common blocks that we can discard due to being inferior chains + int dropPeersAfterCommonBlockHeight = 0; + + // Remove peers with no common block data + Iterator iterator = peers.iterator(); + while (iterator.hasNext()) { + Peer peer = (Peer) iterator.next(); + if (peer.getCommonBlockData() == null) { + LOGGER.debug(String.format("Removed peer %s because it has no common block data", peer)); + iterator.remove(); + } + } + + // Loop through each group of common blocks + for (BlockSummaryData commonBlockSummary : commonBlocks) { + List peersSharingCommonBlock = peers.stream().filter(peer -> peer.getCommonBlockData().getCommonBlockSummary().equals(commonBlockSummary)).collect(Collectors.toList()); + + // Check if we need to discard this group of peers + if (dropPeersAfterCommonBlockHeight > 0) { + if (commonBlockSummary.getHeight() > dropPeersAfterCommonBlockHeight) { + // We have already determined that the correct chain diverged from a lower height. We are safe to skip these peers. + for (Peer peer : peersSharingCommonBlock) { + LOGGER.debug(String.format("Peer %s has common block at height %d but the superior chain is at height %d. Removing it from this round.", peer, commonBlockSummary.getHeight(), dropPeersAfterCommonBlockHeight)); + Controller.getInstance().addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature()); + } + continue; + } + } + + // Calculate the length of the shortest peer chain sharing this common block, including our chain + final int ourAdditionalBlocksAfterCommonBlock = ourHeight - commonBlockSummary.getHeight(); + int minChainLength = this.calculateMinChainLengthOfPeers(peersSharingCommonBlock, commonBlockSummary); + + // Fetch block summaries from each peer + for (Peer peer : peersSharingCommonBlock) { + + // If we're shutting down, just return the latest peer list + if (Controller.isStopping()) + return peers; + + // Count the number of blocks this peer has beyond our common block + final int peerHeight = peer.getChainTipData().getLastHeight(); + final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); + // Limit the number of blocks we are comparing. FUTURE: we could request more in batches, but there may not be a case when this is needed + int summariesRequired = Math.min(peerAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE); + + // Check if we can use the cached common block summaries, by comparing the peer's current chain tip against the peer's chain tip when we last found our common block + boolean useCachedSummaries = false; + if (peer.canUseCachedCommonBlockData()) { + if (peer.getCommonBlockData().getBlockSummariesAfterCommonBlock() != null) { + if (peer.getCommonBlockData().getBlockSummariesAfterCommonBlock().size() == summariesRequired) { + LOGGER.debug(String.format("Using cached block summaries for peer %s", peer)); + useCachedSummaries = true; + } + } + } + + if (useCachedSummaries == false) { + if (summariesRequired > 0) { + LOGGER.trace(String.format("Requesting %d block summar%s from peer %s after common block %.8s. Peer height: %d", summariesRequired, (summariesRequired != 1 ? "ies" : "y"), peer, Base58.encode(commonBlockSummary.getSignature()), peerHeight)); + + List blockSummaries = this.getBlockSummaries(peer, commonBlockSummary.getSignature(), summariesRequired); + peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(blockSummaries); + + if (blockSummaries != null) { + LOGGER.trace(String.format("Peer %s returned %d block summar%s", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"))); + + // We need to adjust minChainLength if peers fail to return all expected block summaries + if (blockSummaries.size() < summariesRequired) { + // This could mean that the peer has re-orged. But we still have the same common block, so it's safe to proceed with this set of signatures instead. + LOGGER.debug(String.format("Peer %s returned %d block summar%s instead of expected %d", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"), summariesRequired)); + + // Reduce minChainLength if we have at least 1 block for this peer. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength + if (blockSummaries.size() > 0) + if (blockSummaries.size() < minChainLength) + minChainLength = blockSummaries.size(); + } + } + } else { + // There are no block summaries after this common block + peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(null); + } + } + } + + // Fetch our corresponding block summaries. Limit to MAXIMUM_REQUEST_SIZE, in order to make the comparison fairer, as peers have been limited too + final int ourSummariesRequired = Math.min(ourAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE); + LOGGER.trace(String.format("About to fetch our block summaries from %d to %d. Our height: %d", commonBlockSummary.getHeight() + 1, commonBlockSummary.getHeight() + ourSummariesRequired, ourHeight)); + List ourBlockSummaries = repository.getBlockRepository().getBlockSummaries(commonBlockSummary.getHeight() + 1, commonBlockSummary.getHeight() + ourSummariesRequired); + if (ourBlockSummaries.isEmpty()) { + LOGGER.debug(String.format("We don't have any block summaries so can't compare our chain against peers with this common block. We can still compare them against each other.")); + } + else { + populateBlockSummariesMinterLevels(repository, ourBlockSummaries); + // Reduce minChainLength if we have less summaries + if (ourBlockSummaries.size() < minChainLength) + minChainLength = ourBlockSummaries.size(); + } + + // Create array to hold peers for comparison + List superiorPeersForComparison = new ArrayList<>(); + + // Calculate max height for chain weight comparisons + int maxHeightForChainWeightComparisons = commonBlockSummary.getHeight() + minChainLength; + + // Calculate our chain weight + BigInteger ourChainWeight = BigInteger.valueOf(0); + if (ourBlockSummaries.size() > 0) + ourChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), ourBlockSummaries, maxHeightForChainWeightComparisons); + + NumberFormat formatter = new DecimalFormat("0.###E0"); + NumberFormat accurateFormatter = new DecimalFormat("0.################E0"); + LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", ourBlockSummaries.size(), formatter.format(ourChainWeight))); + + LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature()))); + for (Peer peer : peersSharingCommonBlock) { + final int peerHeight = peer.getChainTipData().getLastHeight(); + final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); + final CommonBlockData peerCommonBlockData = peer.getCommonBlockData(); + + if (peerCommonBlockData == null || peerCommonBlockData.getBlockSummariesAfterCommonBlock() == null || peerCommonBlockData.getBlockSummariesAfterCommonBlock().isEmpty()) { + // No response - remove this peer for now + LOGGER.debug(String.format("Peer %s doesn't have any block summaries - removing it from this round", peer)); + peers.remove(peer); + continue; + } + + final List peerBlockSummariesAfterCommonBlock = peerCommonBlockData.getBlockSummariesAfterCommonBlock(); + populateBlockSummariesMinterLevels(repository, peerBlockSummariesAfterCommonBlock); + + // Calculate cumulative chain weight of this blockchain subset, from common block to highest mutual block held by all peers in this group. + LOGGER.debug(String.format("About to calculate chain weight based on %d blocks for peer %s with common block %.8s (peer has %d blocks after common block)", peerBlockSummariesAfterCommonBlock.size(), peer, Base58.encode(commonBlockSummary.getSignature()), peerAdditionalBlocksAfterCommonBlock)); + BigInteger peerChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), peerBlockSummariesAfterCommonBlock, maxHeightForChainWeightComparisons); + peer.getCommonBlockData().setChainWeight(peerChainWeight); + LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, peerBlockSummariesAfterCommonBlock.size(), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), formatter.format(peerChainWeight))); + + // Compare against our chain - if our blockchain has greater weight then don't synchronize with peer (or any others in this group) + if (ourChainWeight.compareTo(peerChainWeight) > 0) { + // This peer is on an inferior chain - remove it + LOGGER.debug(String.format("Peer %s is on an inferior chain to us - removing it from this round", peer)); + peers.remove(peer); + } + else { + // Our chain is inferior + LOGGER.debug(String.format("Peer %s is on a better chain to us. We will compare the other peers sharing this common block against each other, and drop all peers sharing higher common blocks.", peer)); + dropPeersAfterCommonBlockHeight = commonBlockSummary.getHeight(); + superiorPeersForComparison.add(peer); + } + } + + // Now that we have selected the best peers, compare them against each other and remove any with lower weights + if (superiorPeersForComparison.size() > 0) { + BigInteger bestChainWeight = null; + for (Peer peer : superiorPeersForComparison) { + // Increase bestChainWeight if needed + if (bestChainWeight == null || peer.getCommonBlockData().getChainWeight().compareTo(bestChainWeight) >= 0) + bestChainWeight = peer.getCommonBlockData().getChainWeight(); + } + for (Peer peer : superiorPeersForComparison) { + // Check if we should discard an inferior peer + if (peer.getCommonBlockData().getChainWeight().compareTo(bestChainWeight) < 0) { + BigInteger difference = bestChainWeight.subtract(peer.getCommonBlockData().getChainWeight()); + LOGGER.debug(String.format("Peer %s has a lower chain weight (difference: %s) than other peer(s) in this group - removing it from this round.", peer, accurateFormatter.format(difference))); + peers.remove(peer); + } + } + } + } + + return peers; + } finally { + repository.discardChanges(); // Free repository locks, if any, also in case anything went wrong + } + } catch (DataException e) { + LOGGER.error("Repository issue during peer comparison", e); + return peers; + } + } + + private List uniqueCommonBlocks(List peers) { + List commonBlocks = new ArrayList<>(); + + for (Peer peer : peers) { + if (peer.getCommonBlockData() != null && peer.getCommonBlockData().getCommonBlockSummary() != null) { + LOGGER.debug(String.format("Peer %s has common block %.8s", peer, Base58.encode(peer.getCommonBlockData().getCommonBlockSummary().getSignature()))); + + BlockSummaryData commonBlockSummary = peer.getCommonBlockData().getCommonBlockSummary(); + if (!commonBlocks.contains(commonBlockSummary)) + commonBlocks.add(commonBlockSummary); + } + else { + LOGGER.debug(String.format("Peer %s has no common block data. Skipping...", peer)); + } + } + + return commonBlocks; + } + + private int calculateMinChainLengthOfPeers(List peersSharingCommonBlock, BlockSummaryData commonBlockSummary) { + // Calculate the length of the shortest peer chain sharing this common block, including our chain + int minChainLength = 0; + for (Peer peer : peersSharingCommonBlock) { + final int peerHeight = peer.getChainTipData().getLastHeight(); + final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); + + if (peerAdditionalBlocksAfterCommonBlock < minChainLength || minChainLength == 0) + minChainLength = peerAdditionalBlocksAfterCommonBlock; + } + return minChainLength; + } + + /** * Attempt to synchronize blockchain with peer. *

@@ -97,9 +483,12 @@ public class Synchronizer { List peerBlockSummaries = new ArrayList<>(); SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, force, peerBlockSummaries); - if (findCommonBlockResult != SynchronizationResult.OK) + if (findCommonBlockResult != SynchronizationResult.OK) { // Logging performed by fetchSummariesFromCommonBlock() above + // Clear our common block cache for this peer + peer.setCommonBlockData(null); return findCommonBlockResult; + } // First summary is common block final BlockData commonBlockData = repository.getBlockRepository().fromSignature(peerBlockSummaries.get(0).getSignature()); @@ -244,9 +633,13 @@ public class Synchronizer { // Currently we work forward from common block until we hit a block we don't have // TODO: rewrite as modified binary search! int i; - for (i = 1; i < blockSummariesFromCommon.size(); ++i) + for (i = 1; i < blockSummariesFromCommon.size(); ++i) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + if (!repository.getBlockRepository().exists(blockSummariesFromCommon.get(i).getSignature())) break; + } // Note: index i - 1 isn't cleared: List.subList is fromIndex inclusive to toIndex exclusive blockSummariesFromCommon.subList(0, i - 1).clear(); @@ -295,6 +688,9 @@ public class Synchronizer { // Check peer sent valid heights for (int i = 0; i < moreBlockSummaries.size(); ++i) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + ++lastSummaryHeight; BlockSummaryData blockSummary = moreBlockSummaries.get(i); @@ -316,7 +712,7 @@ public class Synchronizer { populateBlockSummariesMinterLevels(repository, ourBlockSummaries); populateBlockSummariesMinterLevels(repository, peerBlockSummaries); - final int mutualHeight = commonBlockHeight - 1 + Math.min(ourBlockSummaries.size(), peerBlockSummaries.size()); + final int mutualHeight = commonBlockHeight + Math.min(ourBlockSummaries.size(), peerBlockSummaries.size()); // Calculate cumulative chain weights of both blockchain subsets, from common block to highest mutual block. BigInteger ourChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight); @@ -341,52 +737,154 @@ public class Synchronizer { final byte[] commonBlockSig = commonBlockData.getSignature(); String commonBlockSig58 = Base58.encode(commonBlockSig); + byte[] latestPeerSignature = commonBlockSig; + int height = commonBlockHeight; + LOGGER.debug(() -> String.format("Fetching peer %s chain from height %d, sig %.8s", peer, commonBlockHeight, commonBlockSig58)); - int ourHeight = ourInitialHeight; // Overall plan: fetch peer's blocks first, then orphan, then apply // Convert any leftover (post-common) block summaries into signatures to request from peer List peerBlockSignatures = peerBlockSummaries.stream().map(BlockSummaryData::getSignature).collect(Collectors.toList()); - // Fetch remaining block signatures, if needed - int numberSignaturesRequired = peerBlockSignatures.size() - (peerHeight - commonBlockHeight); - if (numberSignaturesRequired > 0) { - byte[] latestPeerSignature = peerBlockSignatures.isEmpty() ? commonBlockSig : peerBlockSignatures.get(peerBlockSignatures.size() - 1); - - LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s", - numberSignaturesRequired, (numberSignaturesRequired != 1 ? "s": ""), ourHeight, Base58.encode(latestPeerSignature))); - - List moreBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberSignaturesRequired); - - if (moreBlockSignatures == null || moreBlockSignatures.isEmpty()) { - LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, - ourHeight, Base58.encode(latestPeerSignature))); - return SynchronizationResult.NO_REPLY; - } - - LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : ""))); - - peerBlockSignatures.addAll(moreBlockSignatures); - } - - // Fetch blocks using signatures - LOGGER.debug(String.format("Fetching new blocks from peer %s", peer)); + // Keep a list of blocks received so far List peerBlocks = new ArrayList<>(); - for (byte[] blockSignature : peerBlockSignatures) { - Block newBlock = this.fetchBlock(repository, peer, blockSignature); + // Calculate the total number of additional blocks this peer has beyond the common block + int additionalPeerBlocksAfterCommonBlock = peerHeight - commonBlockHeight; + // Subtract the number of signatures that we already have, as we don't need to request them again + int numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size(); + + int retryCount = 0; + while (height < peerHeight) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + // Ensure we don't request more than MAXIMUM_REQUEST_SIZE + int numberRequested = Math.min(numberSignaturesRequired, MAXIMUM_REQUEST_SIZE); + + // Do we need more signatures? + if (peerBlockSignatures.isEmpty() && numberRequested > 0) { + LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s", + numberRequested, (numberRequested != 1 ? "s" : ""), height, Base58.encode(latestPeerSignature))); + + peerBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberRequested); + + if (peerBlockSignatures == null || peerBlockSignatures.isEmpty()) { + LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, + height, Base58.encode(latestPeerSignature))); + + // Clear our cache of common block summaries for this peer, as they are likely to be invalid + CommonBlockData cachedCommonBlockData = peer.getCommonBlockData(); + if (cachedCommonBlockData != null) + cachedCommonBlockData.setBlockSummariesAfterCommonBlock(null); + + // If we have already received recent or newer blocks from this peer, go ahead and apply them + if (peerBlocks.size() > 0) { + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1); + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (ourLatestBlockData != null && peerLatestBlock != null && minLatestBlockTimestamp != null) { + + // If we have received at least one recent block, we can apply them + if (peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) { + LOGGER.debug("Newly received blocks are recent, so we will apply them"); + break; + } + + // If our latest block is very old.... + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + // ... and we have received a block that is more recent than our latest block ... + if (peerLatestBlock.getBlockData().getTimestamp() > ourLatestBlockData.getTimestamp()) { + // ... then apply the blocks, as it takes us a step forward. + // This is particularly useful when starting up a node that was on a small fork when it was last shut down. + // In these cases, we now allow the node to sync forward, and get onto the main chain again. + // Without this, we would require that the node syncs ENTIRELY with this peer, + // and any problems downloading a block would cause all progress to be lost. + LOGGER.debug(String.format("Newly received blocks are %d ms newer than our latest block - so we will apply them", peerLatestBlock.getBlockData().getTimestamp() - ourLatestBlockData.getTimestamp())); + break; + } + } + } + } + // Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated state + return SynchronizationResult.NO_REPLY; + } + + numberSignaturesRequired = peerHeight - height - peerBlockSignatures.size(); + LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : ""))); + } + + if (peerBlockSignatures.isEmpty()) { + LOGGER.trace(String.format("No more signatures or blocks to request from peer %s", peer)); + break; + } + + byte[] nextPeerSignature = peerBlockSignatures.get(0); + int nextHeight = height + 1; + + LOGGER.trace(String.format("Fetching block %d, sig %.8s from %s", nextHeight, Base58.encode(nextPeerSignature), peer)); + Block newBlock = this.fetchBlock(repository, peer, nextPeerSignature); if (newBlock == null) { LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, - ourHeight, Base58.encode(blockSignature))); - return SynchronizationResult.NO_REPLY; + nextHeight, Base58.encode(nextPeerSignature))); + + if (retryCount >= MAXIMUM_RETRIES) { + + // If we have already received recent or newer blocks from this peer, go ahead and apply them + if (peerBlocks.size() > 0) { + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1); + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (ourLatestBlockData != null && peerLatestBlock != null && minLatestBlockTimestamp != null) { + + // If we have received at least one recent block, we can apply them + if (peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) { + LOGGER.debug("Newly received blocks are recent, so we will apply them"); + break; + } + + // If our latest block is very old.... + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + // ... and we have received a block that is more recent than our latest block ... + if (peerLatestBlock.getBlockData().getTimestamp() > ourLatestBlockData.getTimestamp()) { + // ... then apply the blocks, as it takes us a step forward. + // This is particularly useful when starting up a node that was on a small fork when it was last shut down. + // In these cases, we now allow the node to sync forward, and get onto the main chain again. + // Without this, we would require that the node syncs ENTIRELY with this peer, + // and any problems downloading a block would cause all progress to be lost. + LOGGER.debug(String.format("Newly received blocks are %d ms newer than our latest block - so we will apply them", peerLatestBlock.getBlockData().getTimestamp() - ourLatestBlockData.getTimestamp())); + break; + } + } + } + } + // Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated state + return SynchronizationResult.NO_REPLY; + + } else { + // Re-fetch signatures, in case the peer is now on a different fork + peerBlockSignatures.clear(); + numberSignaturesRequired = peerHeight - height; + + // Retry until retryCount reaches MAXIMUM_RETRIES + retryCount++; + int triesRemaining = MAXIMUM_RETRIES - retryCount; + LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s" : ""))); + continue; + } } + // Reset retryCount because the last request succeeded + retryCount = 0; + + LOGGER.trace(String.format("Fetched block %d, sig %.8s from %s", nextHeight, Base58.encode(latestPeerSignature), peer)); + if (!newBlock.isSignatureValid()) { LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer, - ourHeight, Base58.encode(blockSignature))); + nextHeight, Base58.encode(latestPeerSignature))); return SynchronizationResult.INVALID_DATA; } @@ -395,12 +893,18 @@ public class Synchronizer { transaction.setInitialApprovalStatus(); peerBlocks.add(newBlock); + + // Now that we've received this block, we can increase our height and move on to the next one + latestPeerSignature = nextPeerSignature; + peerBlockSignatures.remove(0); + ++height; } // Unwind to common block (unless common block is our latest block) - LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s", commonBlockHeight, commonBlockSig58)); + int ourHeight = ourInitialHeight; + LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s. Our height: %d", commonBlockHeight, commonBlockSig58, ourHeight)); - BlockData orphanBlockData = repository.getBlockRepository().fromHeight(ourHeight); + BlockData orphanBlockData = repository.getBlockRepository().fromHeight(ourInitialHeight); while (ourHeight > commonBlockHeight) { if (Controller.isStopping()) return SynchronizationResult.SHUTTING_DOWN; @@ -422,10 +926,13 @@ public class Synchronizer { LOGGER.debug(String.format("Orphaned blocks back to height %d, sig %.8s - applying new blocks from peer %s", commonBlockHeight, commonBlockSig58, peer)); for (Block newBlock : peerBlocks) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + ValidationResult blockResult = newBlock.isValid(); if (blockResult != ValidationResult.OK) { LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer, - ourHeight, Base58.encode(newBlock.getSignature()), blockResult.name())); + newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getSignature()), blockResult.name())); return SynchronizationResult.INVALID_DATA; } @@ -469,7 +976,8 @@ public class Synchronizer { // Do we need more signatures? if (peerBlockSignatures.isEmpty()) { - int numberRequested = maxBatchHeight - ourHeight; + int numberRequested = Math.min(maxBatchHeight - ourHeight, MAXIMUM_REQUEST_SIZE); + LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s", numberRequested, (numberRequested != 1 ? "s": ""), ourHeight, Base58.encode(latestPeerSignature))); @@ -488,7 +996,9 @@ public class Synchronizer { peerBlockSignatures.remove(0); ++ourHeight; + LOGGER.trace(String.format("Fetching block %d, sig %.8s from %s", ourHeight, Base58.encode(latestPeerSignature), peer)); Block newBlock = this.fetchBlock(repository, peer, latestPeerSignature); + LOGGER.trace(String.format("Fetched block %d, sig %.8s from %s", ourHeight, Base58.encode(latestPeerSignature), peer)); if (newBlock == null) { LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, @@ -571,6 +1081,9 @@ public class Synchronizer { final int firstBlockHeight = blockSummaries.get(0).getHeight(); for (int i = 0; i < blockSummaries.size(); ++i) { + if (Controller.isStopping()) + return; + BlockSummaryData blockSummary = blockSummaries.get(i); // Qortal: minter is always a reward-share, so find actual minter and get their effective minting level diff --git a/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java b/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java index 51b2b075..84a0d484 100644 --- a/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java @@ -23,7 +23,7 @@ public interface AcctTradeBot { public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String foreignKey, String receivingAddress) throws DataException; - public boolean canDelete(Repository repository, TradeBotData tradeBotData); + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException; public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException; diff --git a/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java index fe0f41c1..802a2870 100644 --- a/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java @@ -345,11 +345,15 @@ public class BitcoinACCTv1TradeBot implements AcctTradeBot { } @Override - public boolean canDelete(Repository repository, TradeBotData tradeBotData) { + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { State tradeBotState = State.valueOf(tradeBotData.getStateValue()); if (tradeBotState == null) return true; + // If the AT doesn't exist then we might as well let the user tidy up + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) + return true; + switch (tradeBotState) { case BOB_WAITING_FOR_AT_CONFIRM: case ALICE_DONE: @@ -378,7 +382,16 @@ public class BitcoinACCTv1TradeBot implements AcctTradeBot { // Attempt to fetch AT data atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); if (atData == null) { - LOGGER.warn(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + + // If it has been over 24 hours since we last updated this trade-bot entry then assume AT is never coming back + // and so wipe the trade-bot entry + if (tradeBotData.getTimestamp() + MAX_AT_CONFIRMATION_PERIOD < NTP.getTime()) { + LOGGER.info(() -> String.format("AT %s has been gone for too long - deleting trade-bot entry", tradeBotData.getAtAddress())); + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + repository.saveChanges(); + } + return; } diff --git a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java index 0da3f0ce..286cbf74 100644 --- a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java @@ -211,6 +211,9 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot { TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); + // Attempt to backup the trade bot data + TradeBot.backupTradeBotData(repository); + // Return to user for signing and broadcast as we don't have their Qortal private key try { return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); @@ -283,6 +286,9 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot { tradeForeignPublicKey, tradeForeignPublicKeyHash, crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); + // Attempt to backup the trade bot data + TradeBot.backupTradeBotData(repository); + // Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount long p2shFee; try { @@ -343,11 +349,15 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot { } @Override - public boolean canDelete(Repository repository, TradeBotData tradeBotData) { + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { State tradeBotState = State.valueOf(tradeBotData.getStateValue()); if (tradeBotState == null) return true; + // If the AT doesn't exist then we might as well let the user tidy up + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) + return true; + switch (tradeBotState) { case BOB_WAITING_FOR_AT_CONFIRM: case ALICE_DONE: @@ -376,7 +386,16 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot { // Attempt to fetch AT data atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); if (atData == null) { - LOGGER.warn(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + + // If it has been over 24 hours since we last updated this trade-bot entry then assume AT is never coming back + // and so wipe the trade-bot entry + if (tradeBotData.getTimestamp() + MAX_AT_CONFIRMATION_PERIOD < NTP.getTime()) { + LOGGER.info(() -> String.format("AT %s has been gone for too long - deleting trade-bot entry", tradeBotData.getAtAddress())); + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + repository.saveChanges(); + } + return; } diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index 84e32125..94c7cefb 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -7,6 +7,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.concurrent.locks.ReentrantLock; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -267,6 +268,22 @@ public class TradeBot implements Listener { return secret; } + /*package*/ static void backupTradeBotData(Repository repository) { + // Attempt to backup the trade bot data. This an optional step and doesn't impact trading, so don't throw an exception on failure + try { + LOGGER.info("About to backup trade bot data..."); + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + blockchainLock.lockInterruptibly(); + try { + repository.exportNodeLocalData(true); + } finally { + blockchainLock.unlock(); + } + } catch (InterruptedException | DataException e) { + LOGGER.info(String.format("Failed to obtain blockchain lock when exporting trade bot data: %s", e.getMessage())); + } + } + /** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */ /*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, String newState, int newStateValue, Supplier logMessageSupplier) throws DataException { diff --git a/src/main/java/org/qortal/crosschain/Bitcoin.java b/src/main/java/org/qortal/crosschain/Bitcoin.java index a8c6469a..28275d6a 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoin.java +++ b/src/main/java/org/qortal/crosschain/Bitcoin.java @@ -42,35 +42,32 @@ public class Bitcoin extends Bitcoiny { public Collection getServers() { return Arrays.asList( // Servers chosen on NO BASIS WHATSOEVER from various sources! - new Server("enode.duckdns.org", Server.ConnectionType.SSL, 50002), - new Server("electrumx.ml", Server.ConnectionType.SSL, 50002), - new Server("electrum.bitkoins.nl", Server.ConnectionType.SSL, 50512), - new Server("btc.electroncash.dk", Server.ConnectionType.SSL, 60002), - new Server("electrumx.electricnewyear.net", Server.ConnectionType.SSL, 50002), - new Server("dxm.no-ip.biz", Server.ConnectionType.TCP, 50001), - new Server("kirsche.emzy.de", Server.ConnectionType.TCP, 50001), - new Server("2AZZARITA.hopto.org", Server.ConnectionType.TCP, 50001), - new Server("xtrum.com", Server.ConnectionType.TCP, 50001), - new Server("electrum.srvmin.network", Server.ConnectionType.TCP, 50001), - new Server("electrumx.alexridevski.net", Server.ConnectionType.TCP, 50001), - new Server("bitcoin.lukechilds.co", Server.ConnectionType.TCP, 50001), - new Server("electrum.poiuty.com", Server.ConnectionType.TCP, 50001), - new Server("horsey.cryptocowboys.net", Server.ConnectionType.TCP, 50001), + new Server("128.0.190.26", Server.ConnectionType.SSL, 50002), + new Server("hodlers.beer", Server.ConnectionType.SSL, 50002), + new Server("electrumx.erbium.eu", Server.ConnectionType.TCP, 50001), + new Server("electrumx.erbium.eu", Server.ConnectionType.SSL, 50002), + new Server("btc.lastingcoin.net", Server.ConnectionType.SSL, 50002), + new Server("electrum.bitaroo.net", Server.ConnectionType.SSL, 50002), + new Server("bitcoin.grey.pw", Server.ConnectionType.SSL, 50002), + new Server("2electrumx.hopto.me", Server.ConnectionType.SSL, 56022), + new Server("185.64.116.15", Server.ConnectionType.SSL, 50002), + new Server("kirsche.emzy.de", Server.ConnectionType.SSL, 50002), + new Server("alviss.coinjoined.com", Server.ConnectionType.SSL, 50002), + new Server("electrum.emzy.de", Server.ConnectionType.SSL, 50002), new Server("electrum.emzy.de", Server.ConnectionType.TCP, 50001), - new Server("electrum-server.ninja", Server.ConnectionType.TCP, 50081), - new Server("bitcoin.electrumx.multicoin.co", Server.ConnectionType.TCP, 50001), - new Server("esx.geekhosters.com", Server.ConnectionType.TCP, 50001), - new Server("bitcoin.grey.pw", Server.ConnectionType.TCP, 50003), - new Server("exs.ignorelist.com", Server.ConnectionType.TCP, 50001), - new Server("electrum.coinext.com.br", Server.ConnectionType.TCP, 50001), - new Server("bitcoin.aranguren.org", Server.ConnectionType.TCP, 50001), - new Server("skbxmit.coinjoined.com", Server.ConnectionType.TCP, 50001), - new Server("alviss.coinjoined.com", Server.ConnectionType.TCP, 50001), - new Server("electrum2.privateservers.network", Server.ConnectionType.TCP, 50001), - new Server("electrumx.schulzemic.net", Server.ConnectionType.TCP, 50001), - new Server("bitcoins.sk", Server.ConnectionType.TCP, 56001), - new Server("node.mendonca.xyz", Server.ConnectionType.TCP, 50001), - new Server("bitcoin.aranguren.org", Server.ConnectionType.TCP, 50001)); + new Server("vmd71287.contaboserver.net", Server.ConnectionType.SSL, 50002), + new Server("btc.litepay.ch", Server.ConnectionType.SSL, 50002), + new Server("electrum.stippy.com", Server.ConnectionType.SSL, 50002), + new Server("xtrum.com", Server.ConnectionType.SSL, 50002), + new Server("electrum.acinq.co", Server.ConnectionType.SSL, 50002), + new Server("electrum2.taborsky.cz", Server.ConnectionType.SSL, 50002), + new Server("vmd63185.contaboserver.net", Server.ConnectionType.SSL, 50002), + new Server("electrum2.privateservers.network", Server.ConnectionType.SSL, 50002), + new Server("electrumx.alexridevski.net", Server.ConnectionType.SSL, 50002), + new Server("192.166.219.200", Server.ConnectionType.SSL, 50002), + new Server("2ex.digitaleveryware.com", Server.ConnectionType.SSL, 50002), + new Server("dxm.no-ip.biz", Server.ConnectionType.SSL, 50002), + new Server("caleb.vegas", Server.ConnectionType.SSL, 50002)); } @Override @@ -96,10 +93,8 @@ public class Bitcoin extends Bitcoiny { @Override public Collection getServers() { return Arrays.asList( - new Server("electrum.blockstream.info", Server.ConnectionType.TCP, 60001), - new Server("electrum.blockstream.info", Server.ConnectionType.SSL, 60002), + new Server("tn.not.fyi", Server.ConnectionType.SSL, 55002), new Server("electrumx-test.1209k.com", Server.ConnectionType.SSL, 50002), - new Server("testnet.qtornado.com", Server.ConnectionType.TCP, 51001), new Server("testnet.qtornado.com", Server.ConnectionType.SSL, 51002), new Server("testnet.aranguren.org", Server.ConnectionType.TCP, 51001), new Server("testnet.aranguren.org", Server.ConnectionType.SSL, 51002), diff --git a/src/main/java/org/qortal/data/block/BlockSummaryData.java b/src/main/java/org/qortal/data/block/BlockSummaryData.java index a0c39f75..2167f0f0 100644 --- a/src/main/java/org/qortal/data/block/BlockSummaryData.java +++ b/src/main/java/org/qortal/data/block/BlockSummaryData.java @@ -2,6 +2,7 @@ package org.qortal.data.block; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Arrays; @XmlAccessorType(XmlAccessType.FIELD) public class BlockSummaryData { @@ -84,4 +85,21 @@ public class BlockSummaryData { this.minterLevel = minterLevel; } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + BlockSummaryData otherBlockSummary = (BlockSummaryData) o; + if (this.getSignature() == null || otherBlockSummary.getSignature() == null) + return false; + + // Treat two block summaries as equal if they have matching signatures + return Arrays.equals(this.getSignature(), otherBlockSummary.getSignature()); + } + } diff --git a/src/main/java/org/qortal/data/block/CommonBlockData.java b/src/main/java/org/qortal/data/block/CommonBlockData.java new file mode 100644 index 00000000..dd502df7 --- /dev/null +++ b/src/main/java/org/qortal/data/block/CommonBlockData.java @@ -0,0 +1,56 @@ +package org.qortal.data.block; + +import org.qortal.data.network.PeerChainTipData; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.math.BigInteger; +import java.util.List; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CommonBlockData { + + // Properties + private BlockSummaryData commonBlockSummary = null; + private List blockSummariesAfterCommonBlock = null; + private BigInteger chainWeight = null; + private PeerChainTipData chainTipData = null; + + // Constructors + + protected CommonBlockData() { + } + + public CommonBlockData(BlockSummaryData commonBlockSummary, PeerChainTipData chainTipData) { + this.commonBlockSummary = commonBlockSummary; + this.chainTipData = chainTipData; + } + + + // Getters / setters + + public BlockSummaryData getCommonBlockSummary() { + return this.commonBlockSummary; + } + + public List getBlockSummariesAfterCommonBlock() { + return this.blockSummariesAfterCommonBlock; + } + + public void setBlockSummariesAfterCommonBlock(List blockSummariesAfterCommonBlock) { + this.blockSummariesAfterCommonBlock = blockSummariesAfterCommonBlock; + } + + public BigInteger getChainWeight() { + return this.chainWeight; + } + + public void setChainWeight(BigInteger chainWeight) { + this.chainWeight = chainWeight; + } + + public PeerChainTipData getChainTipData() { + return this.chainTipData; + } + +} diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index ab8aa1f4..08db0dd9 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -15,6 +15,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Random; +import java.util.Arrays; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -22,6 +23,7 @@ import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.data.block.CommonBlockData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; import org.qortal.network.message.ChallengeMessage; @@ -46,7 +48,7 @@ public class Peer { private static final int CONNECT_TIMEOUT = 2000; // ms /** Maximum time to wait for a message reply to arrive from peer. (ms) */ - private static final int RESPONSE_TIMEOUT = 2000; // ms + private static final int RESPONSE_TIMEOUT = 3000; // ms /** * Interval between PING messages to a peer. (ms) @@ -106,6 +108,9 @@ public class Peer { /** Latest block info as reported by peer. */ private PeerChainTipData peersChainTipData; + /** Our common block with this peer */ + private CommonBlockData commonBlockData; + // Constructors /** Construct unconnected, outbound Peer using socket address in peer data */ @@ -272,6 +277,18 @@ public class Peer { } } + public CommonBlockData getCommonBlockData() { + synchronized (this.peerInfoLock) { + return this.commonBlockData; + } + } + + public void setCommonBlockData(CommonBlockData commonBlockData) { + synchronized (this.peerInfoLock) { + this.commonBlockData = commonBlockData; + } + } + /*package*/ void queueMessage(Message message) { if (!this.pendingMessages.offer(message)) LOGGER.info(() -> String.format("No room to queue message from peer %s - discarding", this)); @@ -507,6 +524,7 @@ public class Peer { } } catch (MessageException e) { LOGGER.warn(String.format("Failed to send %s message with ID %d to peer %s: %s", message.getType().name(), message.getId(), this, e.getMessage())); + return false; } catch (IOException e) { // Send failure return false; @@ -615,6 +633,25 @@ public class Peer { } } + + // Common block data + + public boolean canUseCachedCommonBlockData() { + PeerChainTipData peerChainTipData = this.getChainTipData(); + CommonBlockData commonBlockData = this.getCommonBlockData(); + + if (peerChainTipData != null && commonBlockData != null) { + PeerChainTipData commonBlockChainTipData = commonBlockData.getChainTipData(); + if (peerChainTipData.getLastBlockSignature() != null && commonBlockChainTipData != null && commonBlockChainTipData.getLastBlockSignature() != null) { + if (Arrays.equals(peerChainTipData.getLastBlockSignature(), commonBlockChainTipData.getLastBlockSignature())) { + return true; + } + } + } + return false; + } + + // Utility methods /** Returns true if ports and addresses (or hostnames) match */ diff --git a/src/main/java/org/qortal/network/message/CachedBlockMessage.java b/src/main/java/org/qortal/network/message/CachedBlockMessage.java new file mode 100644 index 00000000..7a175810 --- /dev/null +++ b/src/main/java/org/qortal/network/message/CachedBlockMessage.java @@ -0,0 +1,70 @@ +package org.qortal.network.message; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; + +import org.qortal.block.Block; +import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformer; + +import com.google.common.primitives.Ints; + +// This is an OUTGOING-only Message which more readily lends itself to being cached +public class CachedBlockMessage extends Message { + + private Block block = null; + private byte[] cachedBytes = null; + + public CachedBlockMessage(Block block) { + super(MessageType.BLOCK); + + this.block = block; + } + + private CachedBlockMessage(byte[] cachedBytes) { + super(MessageType.BLOCK); + + this.block = null; + this.cachedBytes = cachedBytes; + } + + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { + throw new UnsupportedOperationException("CachedBlockMessage is for outgoing messages only"); + } + + @Override + protected byte[] toData() { + // Already serialized? + if (this.cachedBytes != null) + return cachedBytes; + + if (this.block == null) + return null; + + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(Ints.toByteArray(this.block.getBlockData().getHeight())); + + bytes.write(BlockTransformer.toBytes(this.block)); + + this.cachedBytes = bytes.toByteArray(); + // We no longer need source Block + // and Block contains repository handle which is highly likely to be invalid after this call + this.block = null; + + return this.cachedBytes; + } catch (TransformationException | IOException e) { + return null; + } + } + + public CachedBlockMessage cloneWithNewId(int newId) { + CachedBlockMessage clone = new CachedBlockMessage(this.cachedBytes); + clone.setId(newId); + return clone; + } + +} diff --git a/src/main/java/org/qortal/repository/Repository.java b/src/main/java/org/qortal/repository/Repository.java index d4ef35ce..5438f1d9 100644 --- a/src/main/java/org/qortal/repository/Repository.java +++ b/src/main/java/org/qortal/repository/Repository.java @@ -47,11 +47,9 @@ public interface Repository extends AutoCloseable { public void backup(boolean quick) throws DataException; - public void checkpoint(boolean quick) throws DataException; - public void performPeriodicMaintenance() throws DataException; - public void exportNodeLocalData() throws DataException; + public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException; public void importDataFromFile(String filename) throws DataException; diff --git a/src/main/java/org/qortal/repository/RepositoryFactory.java b/src/main/java/org/qortal/repository/RepositoryFactory.java index e5b29d1b..bb34d1c9 100644 --- a/src/main/java/org/qortal/repository/RepositoryFactory.java +++ b/src/main/java/org/qortal/repository/RepositoryFactory.java @@ -1,5 +1,7 @@ package org.qortal.repository; +import java.sql.SQLException; + public interface RepositoryFactory { public boolean wasPristineAtOpen(); @@ -12,4 +14,7 @@ public interface RepositoryFactory { public void close() throws DataException; + // Not ideal place for this but implementating class will know the answer without having to open a new DB session + public boolean isDeadlockException(SQLException e); + } diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index 2b6e637b..df578888 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -1,9 +1,14 @@ package org.qortal.repository; +import java.sql.SQLException; + public abstract class RepositoryManager { private static RepositoryFactory repositoryFactory = null; + /** null if no checkpoint requested, TRUE for quick checkpoint, false for slow/full checkpoint. */ + private static Boolean quickCheckpointRequested = null; + public static RepositoryFactory getRepositoryFactory() { return repositoryFactory; } @@ -46,12 +51,12 @@ public abstract class RepositoryManager { } } - public static void checkpoint(boolean quick) { - try (final Repository repository = getRepository()) { - repository.checkpoint(quick); - } catch (DataException e) { - // Checkpoint is best-effort so don't complain - } + public static void setRequestedCheckpoint(Boolean quick) { + quickCheckpointRequested = quick; + } + + public static Boolean getRequestedCheckpoint() { + return quickCheckpointRequested; } public static void rebuild() throws DataException { @@ -66,4 +71,10 @@ public abstract class RepositoryManager { repositoryFactory = oldRepositoryFactory.reopen(); } + public static boolean isDeadlockRelated(Throwable e) { + Throwable cause = e.getCause(); + + return SQLException.class.isInstance(cause) && repositoryFactory.isDeadlockException((SQLException) cause); + } + } diff --git a/src/main/java/org/qortal/repository/TransactionRepository.java b/src/main/java/org/qortal/repository/TransactionRepository.java index acde78df..4e5999eb 100644 --- a/src/main/java/org/qortal/repository/TransactionRepository.java +++ b/src/main/java/org/qortal/repository/TransactionRepository.java @@ -1,5 +1,6 @@ package org.qortal.repository; +import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -251,6 +252,14 @@ public interface TransactionRepository { */ public List getUnconfirmedTransactions(TransactionType txType, byte[] creatorPublicKey) throws DataException; + /** + * Returns list of unconfirmed transactions excluding specified type(s). + * + * @return list of transactions, or empty if none. + * @throws DataException + */ + public List getUnconfirmedTransactions(EnumSet excludedTxTypes) throws DataException; + /** * Remove transaction from unconfirmed transactions pile. * diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 563038d7..5557c13e 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -1,5 +1,6 @@ package org.qortal.repository.hsqldb; +import java.awt.TrayIcon.MessageType; import java.io.File; import java.io.IOException; import java.math.BigDecimal; @@ -31,6 +32,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.account.PrivateKeyAccount; import org.qortal.crypto.Crypto; +import org.qortal.globalization.Translator; +import org.qortal.gui.SysTray; import org.qortal.repository.ATRepository; import org.qortal.repository.AccountRepository; import org.qortal.repository.ArbitraryRepository; @@ -49,11 +52,17 @@ import org.qortal.repository.TransactionRepository; import org.qortal.repository.VotingRepository; import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository; import org.qortal.settings.Settings; +import org.qortal.utils.NTP; public class HSQLDBRepository implements Repository { private static final Logger LOGGER = LogManager.getLogger(HSQLDBRepository.class); + private static final Object CHECKPOINT_LOCK = new Object(); + + // "serialization failure" + private static final Integer DEADLOCK_ERROR_CODE = Integer.valueOf(-4861); + protected Connection connection; protected final Deque savepoints = new ArrayDeque<>(3); protected boolean debugState = false; @@ -103,7 +112,10 @@ public class HSQLDBRepository implements Repository { throw new DataException("Unable to fetch session ID from repository", e); } - assertEmptyTransaction("connection creation"); + // synchronize to block new connections if checkpointing in progress + synchronized (CHECKPOINT_LOCK) { + assertEmptyTransaction("connection creation"); + } } // Getters / setters @@ -284,6 +296,9 @@ public class HSQLDBRepository implements Repository { this.sqlStatements = null; this.savepoints.clear(); + // If a checkpoint has been requested, we could perform that now + this.maybeCheckpoint(); + // Give connection back to the pool this.connection.close(); this.connection = null; @@ -292,6 +307,58 @@ public class HSQLDBRepository implements Repository { } } + private void maybeCheckpoint() throws DataException { + // To serialize checkpointing and to block new sessions when checkpointing in progress + synchronized (CHECKPOINT_LOCK) { + Boolean quickCheckpointRequest = RepositoryManager.getRequestedCheckpoint(); + if (quickCheckpointRequest == null) + return; + + // We can only perform a CHECKPOINT if no other HSQLDB session is mid-transaction, + // otherwise the CHECKPOINT blocks for COMMITs and other threads can't open HSQLDB sessions + // due to HSQLDB blocking until CHECKPOINT finishes - i.e. deadlock + String sql = "SELECT COUNT(*) " + + "FROM Information_schema.system_sessions " + + "WHERE transaction = TRUE"; + + try { + PreparedStatement pstmt = this.cachePreparedStatement(sql); + + if (!pstmt.execute()) + throw new DataException("Unable to check repository session status"); + + try (ResultSet resultSet = pstmt.getResultSet()) { + if (resultSet == null || !resultSet.next()) + // Failed to even find HSQLDB session info! + throw new DataException("No results when checking repository session status"); + + int transactionCount = resultSet.getInt(1); + + if (transactionCount > 0) + // We can't safely perform CHECKPOINT due to ongoing SQL transactions + return; + } + + LOGGER.info("Performing repository CHECKPOINT..."); + + if (Settings.getInstance().getShowCheckpointNotification()) + SysTray.getInstance().showMessage(Translator.INSTANCE.translate("SysTray", "DB_CHECKPOINT"), + Translator.INSTANCE.translate("SysTray", "PERFORMING_DB_CHECKPOINT"), + MessageType.INFO); + + try (Statement stmt = this.connection.createStatement()) { + stmt.execute(Boolean.TRUE.equals(quickCheckpointRequest) ? "CHECKPOINT" : "CHECKPOINT DEFRAG"); + } + + // Completed! + LOGGER.info("Repository CHECKPOINT completed!"); + RepositoryManager.setRequestedCheckpoint(null); + } catch (SQLException e) { + throw new DataException("Unable to check repository session status", e); + } + } + } + @Override public void rebuild() throws DataException { LOGGER.info("Rebuilding repository from scratch"); @@ -379,15 +446,6 @@ public class HSQLDBRepository implements Repository { } } - @Override - public void checkpoint(boolean quick) throws DataException { - try (Statement stmt = this.connection.createStatement()) { - stmt.execute(quick ? "CHECKPOINT" : "CHECKPOINT DEFRAG"); - } catch (SQLException e) { - throw new DataException("Unable to perform repository checkpoint"); - } - } - @Override public void performPeriodicMaintenance() throws DataException { // Defrag DB - takes a while! @@ -402,10 +460,44 @@ public class HSQLDBRepository implements Repository { } @Override - public void exportNodeLocalData() throws DataException { + public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException { + + // Create the qortal-backup folder if it doesn't exist + Path backupPath = Paths.get("qortal-backup"); + try { + Files.createDirectories(backupPath); + } catch (IOException e) { + LOGGER.info("Unable to create backup folder"); + throw new DataException("Unable to create backup folder"); + } + + // We need to rename or delete an existing TradeBotStates backup before creating a new one + File tradeBotStatesBackupFile = new File("qortal-backup/TradeBotStates.script"); + if (tradeBotStatesBackupFile.exists()) { + if (keepArchivedCopy) { + // Rename existing TradeBotStates backup, to make sure that we're not overwriting any keys + File archivedBackupFile = new File(String.format("qortal-backup/TradeBotStates-archive-%d.script", NTP.getTime())); + if (tradeBotStatesBackupFile.renameTo(archivedBackupFile)) + LOGGER.info(String.format("Moved existing TradeBotStates backup file to %s", archivedBackupFile.getPath())); + else + throw new DataException("Unable to rename existing TradeBotStates backup"); + } else { + // Delete existing copy + LOGGER.info("Deleting existing TradeBotStates backup because it is being replaced with a new one"); + tradeBotStatesBackupFile.delete(); + } + } + + // There's currently no need to take an archived copy of the MintingAccounts data - just delete the old one if it exists + File mintingAccountsBackupFile = new File("qortal-backup/MintingAccounts.script"); + if (mintingAccountsBackupFile.exists()) { + LOGGER.info("Deleting existing MintingAccounts backup because it is being replaced with a new one"); + mintingAccountsBackupFile.delete(); + } + try (Statement stmt = this.connection.createStatement()) { - stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'MintingAccounts.script'"); - stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'TradeBotStates.script'"); + stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'qortal-backup/MintingAccounts.script'"); + stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'qortal-backup/TradeBotStates.script'"); LOGGER.info("Exported sensitive/node-local data: minting keys and trade bot states"); } catch (SQLException e) { throw new DataException("Unable to export sensitive/node-local data from repository"); @@ -418,12 +510,12 @@ public class HSQLDBRepository implements Repository { LOGGER.info(() -> String.format("Importing data into repository from %s", filename)); String escapedFilename = stmt.enquoteLiteral(filename); - stmt.execute("PERFORM IMPORT SCRIPT DATA FROM " + escapedFilename + " STOP ON ERROR"); + stmt.execute("PERFORM IMPORT SCRIPT DATA FROM " + escapedFilename + " CONTINUE ON ERROR"); LOGGER.info(() -> String.format("Imported data into repository from %s", filename)); } catch (SQLException e) { LOGGER.info(() -> String.format("Failed to import data into repository from %s: %s", filename, e.getMessage())); - throw new DataException("Unable to export sensitive/node-local data from repository: " + e.getMessage()); + throw new DataException("Unable to import sensitive/node-local data to repository: " + e.getMessage()); } } @@ -624,7 +716,7 @@ public class HSQLDBRepository implements Repository { /** * Execute PreparedStatement and return changed row count. * - * @param preparedStatement + * @param sql * @param objects * @return number of changed rows * @throws SQLException @@ -636,8 +728,8 @@ public class HSQLDBRepository implements Repository { /** * Execute batched PreparedStatement * - * @param preparedStatement - * @param objects + * @param sql + * @param batchedObjects * @return number of changed rows * @throws SQLException */ @@ -654,7 +746,16 @@ public class HSQLDBRepository implements Repository { long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis(); - int[] updateCounts = preparedStatement.executeBatch(); + int[] updateCounts = null; + try { + updateCounts = preparedStatement.executeBatch(); + } catch (SQLException e) { + if (isDeadlockException(e)) + // We want more info on what other DB sessions are doing to cause this + examineException(e); + + throw e; + } if (this.slowQueryThreshold != null) { long queryTime = System.currentTimeMillis() - beforeQuery; @@ -752,7 +853,7 @@ public class HSQLDBRepository implements Repository { * * @param tableName * @param whereClause - * @param objects + * @param batchedObjects * @throws SQLException */ public int deleteBatch(String tableName, String whereClause, List batchedObjects) throws SQLException { @@ -865,6 +966,8 @@ public class HSQLDBRepository implements Repository { /** Logs other HSQLDB sessions then returns passed exception */ public SQLException examineException(SQLException e) { + // TODO: could log at DEBUG for deadlocks by checking RepositoryManager.isDeadlockRelated(e)? + LOGGER.error(() -> String.format("[Session %d] HSQLDB error: %s", this.sessionId, e.getMessage()), e); logStatements(); @@ -946,4 +1049,8 @@ public class HSQLDBRepository implements Repository { return Crypto.toAddress(publicKey); } + /*package*/ static boolean isDeadlockException(SQLException e) { + return DEADLOCK_ERROR_CODE.equals(e.getErrorCode()); + } + } \ No newline at end of file diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java index 81bf320b..be9c09eb 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java @@ -14,11 +14,11 @@ import org.hsqldb.jdbc.HSQLDBPool; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryFactory; +import org.qortal.settings.Settings; public class HSQLDBRepositoryFactory implements RepositoryFactory { private static final Logger LOGGER = LogManager.getLogger(HSQLDBRepositoryFactory.class); - private static final int POOL_SIZE = 100; /** Log getConnection() calls that take longer than this. (ms) */ private static final long SLOW_CONNECTION_THRESHOLD = 1000L; @@ -57,7 +57,7 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory { HSQLDBRepository.attemptRecovery(connectionUrl); } - this.connectionPool = new HSQLDBPool(POOL_SIZE); + this.connectionPool = new HSQLDBPool(Settings.getInstance().getRepositoryConnectionPoolSize()); this.connectionPool.setUrl(this.connectionUrl); Properties properties = new Properties(); @@ -94,7 +94,11 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory { @Override public Repository tryRepository() throws DataException { try { - return new HSQLDBRepository(this.tryConnection()); + Connection connection = this.tryConnection(); + if (connection == null) + return null; + + return new HSQLDBRepository(connection); } catch (SQLException e) { throw new DataException("Repository instantiation error", e); } @@ -144,4 +148,9 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory { } } + @Override + public boolean isDeadlockException(SQLException e) { + return HSQLDBRepository.isDeadlockException(e); + } + } diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index 83eeba72..a8062e2d 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -9,6 +9,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.EnumMap; +import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -1181,6 +1182,51 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } } + @Override + public List getUnconfirmedTransactions(EnumSet excludedTxTypes) throws DataException { + StringBuilder sql = new StringBuilder(1024); + sql.append("SELECT signature FROM UnconfirmedTransactions "); + sql.append("JOIN Transactions USING (signature) "); + sql.append("WHERE type NOT IN ("); + + boolean firstTxType = true; + for (TransactionType txType : excludedTxTypes) { + if (firstTxType) + firstTxType = false; + else + sql.append(", "); + + sql.append(txType.value); + } + + sql.append(")"); + sql.append("ORDER BY created_when, signature"); + + List transactions = new ArrayList<>(); + + // Find transactions with no corresponding row in BlockTransactions + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) { + if (resultSet == null) + return transactions; + + do { + byte[] signature = resultSet.getBytes(1); + + TransactionData transactionData = this.fromSignature(signature); + + if (transactionData == null) + // Something inconsistent with the repository + throw new DataException(String.format("Unable to fetch unconfirmed transaction %s from repository?", Base58.encode(signature))); + + transactions.add(transactionData); + } while (resultSet.next()); + + return transactions; + } catch (SQLException | DataException e) { + throw new DataException("Unable to fetch unconfirmed transactions from repository", e); + } + } + @Override public void confirmTransaction(byte[] signature) throws DataException { try { diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 22d8da24..cf1830cf 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -52,7 +52,7 @@ public class Settings { // UI servers private int uiPort = 12388; private String[] uiLocalServers = new String[] { - "localhost", "127.0.0.1", "172.24.1.1", "qor.tal" + "localhost", "127.0.0.1" }; private String[] uiRemoteServers = new String[] { "node1.qortal.org", "node2.qortal.org", "node3.qortal.org", "node4.qortal.org", "node5.qortal.org", @@ -89,6 +89,8 @@ public class Settings { private long repositoryCheckpointInterval = 60 * 60 * 1000L; // 1 hour (ms) default /** Whether to show a notification when we perform repository 'checkpoint'. */ private boolean showCheckpointNotification = false; + /* How many blocks to cache locally. Defaulted to 10, which covers a typical Synchronizer request + a few spare */ + private int blockCacheSize = 10; /** How long to keep old, full, AT state data (ms). */ private long atStatesMaxLifetime = 2 * 7 * 24 * 60 * 60 * 1000L; // milliseconds @@ -134,6 +136,8 @@ public class Settings { private Long slowQueryThreshold = null; /** Repository storage path. */ private String repositoryPath = "db"; + /** Repository connection pool size. Needs to be a bit bigger than maxNetworkThreadPoolSize */ + private int repositoryConnectionPoolSize = 100; // Auto-update sources private String[] autoUpdateRepos = new String[] { @@ -361,6 +365,10 @@ public class Settings { return this.maxTransactionTimestampFuture; } + public int getBlockCacheSize() { + return this.blockCacheSize; + } + public boolean isTestNet() { return this.isTestNet; } @@ -424,6 +432,10 @@ public class Settings { return this.repositoryPath; } + public int getRepositoryConnectionPoolSize() { + return this.repositoryConnectionPoolSize; + } + public boolean isAutoUpdateEnabled() { return this.autoUpdateEnabled; } diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index 3ed02f8d..d7dd1455 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -4,6 +4,7 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.EnumSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -605,7 +606,8 @@ public abstract class Transaction { public static List getUnconfirmedTransactions(Repository repository) throws DataException { BlockData latestBlockData = repository.getBlockRepository().getLastBlock(); - List unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(); + EnumSet excludedTxTypes = EnumSet.of(TransactionType.CHAT, TransactionType.PRESENCE); + List unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes); unconfirmedTransactions.sort(getDataComparator()); diff --git a/src/main/java/org/qortal/transform/block/BlockTransformer.java b/src/main/java/org/qortal/transform/block/BlockTransformer.java index fcc0bcad..8b91fd11 100644 --- a/src/main/java/org/qortal/transform/block/BlockTransformer.java +++ b/src/main/java/org/qortal/transform/block/BlockTransformer.java @@ -326,24 +326,36 @@ public class BlockTransformer extends Transformer { } } - public static byte[] getMinterSignatureFromReference(byte[] blockReference) { - return Arrays.copyOf(blockReference, MINTER_SIGNATURE_LENGTH); + private static byte[] getReferenceBytesForMinterSignature(int blockHeight, byte[] reference) { + int newBlockSigTriggerHeight = BlockChain.getInstance().getNewBlockSigHeight(); + + return blockHeight >= newBlockSigTriggerHeight + // 'new' block sig uses all of previous block's signature + ? reference + // 'old' block sig only uses first 64 bytes of previous block's signature + : Arrays.copyOf(reference, MINTER_SIGNATURE_LENGTH); } - public static byte[] getBytesForMinterSignature(BlockData blockData) throws TransformationException { - byte[] minterSignature = getMinterSignatureFromReference(blockData.getReference()); + public static byte[] getBytesForMinterSignature(BlockData blockData) { + byte[] referenceBytes = getReferenceBytesForMinterSignature(blockData.getHeight(), blockData.getReference()); - return getBytesForMinterSignature(minterSignature, blockData.getMinterPublicKey(), blockData.getEncodedOnlineAccounts()); + return getBytesForMinterSignature(referenceBytes, blockData.getMinterPublicKey(), blockData.getEncodedOnlineAccounts()); } - public static byte[] getBytesForMinterSignature(byte[] minterSignature, byte[] minterPublicKey, byte[] encodedOnlineAccounts) { - byte[] bytes = new byte[MINTER_SIGNATURE_LENGTH + MINTER_PUBLIC_KEY_LENGTH + encodedOnlineAccounts.length]; + public static byte[] getBytesForMinterSignature(BlockData parentBlockData, byte[] minterPublicKey, byte[] encodedOnlineAccounts) { + byte[] referenceBytes = getReferenceBytesForMinterSignature(parentBlockData.getHeight() + 1, parentBlockData.getSignature()); - System.arraycopy(minterSignature, 0, bytes, 0, MINTER_SIGNATURE_LENGTH); + return getBytesForMinterSignature(referenceBytes, minterPublicKey, encodedOnlineAccounts); + } - System.arraycopy(minterPublicKey, 0, bytes, MINTER_SIGNATURE_LENGTH, MINTER_PUBLIC_KEY_LENGTH); + private static byte[] getBytesForMinterSignature(byte[] referenceBytes, byte[] minterPublicKey, byte[] encodedOnlineAccounts) { + byte[] bytes = new byte[referenceBytes.length + MINTER_PUBLIC_KEY_LENGTH + encodedOnlineAccounts.length]; - System.arraycopy(encodedOnlineAccounts, 0, bytes, MINTER_SIGNATURE_LENGTH + MINTER_PUBLIC_KEY_LENGTH, encodedOnlineAccounts.length); + System.arraycopy(referenceBytes, 0, bytes, 0, referenceBytes.length); + + System.arraycopy(minterPublicKey, 0, bytes, referenceBytes.length, MINTER_PUBLIC_KEY_LENGTH); + + System.arraycopy(encodedOnlineAccounts, 0, bytes, referenceBytes.length + MINTER_PUBLIC_KEY_LENGTH, encodedOnlineAccounts.length); return bytes; } diff --git a/src/main/resources/.DS_Store b/src/main/resources/.DS_Store new file mode 100644 index 00000000..33653dbd Binary files /dev/null and b/src/main/resources/.DS_Store differ diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 29bb6d1a..8cc06651 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -49,6 +49,8 @@ }, "featureTriggers": { "atFindNextTransactionFix": 275000, + "newBlockSigHeight": 320000, + "shareBinFix": 399000, "calcChainWeightTimestamp": 1616000000000 }, "genesisInfo": { diff --git a/src/main/resources/i18n/ApiError_fi.properties b/src/main/resources/i18n/ApiError_fi.properties new file mode 100644 index 00000000..f9fedf09 --- /dev/null +++ b/src/main/resources/i18n/ApiError_fi.properties @@ -0,0 +1,71 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# Keys are from api.ApiError enum +# +# Kielen muuttaminen suomeksi tapahtuu settings.json-tiedostossa +# +# "localeLang": "fi", +# muista pilkku lopussa jos komento ei ole viimeisellä rivillä + +ADDRESS_UNKNOWN = tilin osoite on tuntematon + +BLOCKCHAIN_NEEDS_SYNC = lohkoketjun tarvitsee ensin synkronisoitua + +# Blocks +BLOCK_UNKNOWN = tuntematon lohko + +BTC_BALANCE_ISSUE = riittämätön Bitcoin-saldo + +BTC_NETWORK_ISSUE = Bitcoin/ElectrumX -verkon ongelma + +BTC_TOO_SOON = liian aikaista julkistaa Bitcoin-tapahtumaa (lukitusaika/mediiaanilohkoaika) + +CANNOT_MINT = tili ei voi lyödä rahaa + +GROUP_UNKNOWN = tuntematon ryhmä + +INVALID_ADDRESS = osoite on kelvoton + +# Assets +INVALID_ASSET_ID = kelvoton ID resurssille + +INVALID_CRITERIA = kelvoton hakuehto + +INVALID_DATA = kelvoton data + +INVALID_HEIGHT = kelvoton lohkon korkeus + +INVALID_NETWORK_ADDRESS = kelvoton verkko-osoite + +INVALID_ORDER_ID = kelvoton resurssin tilaus-ID + +INVALID_PRIVATE_KEY = kelvoton yksityinen avain + +INVALID_PUBLIC_KEY = kelvoton julkinen avain + +INVALID_REFERENCE = kelvoton viite + +# Validation +INVALID_SIGNATURE = kelvoton allekirjoitus + +JSON = JSON-viestin jaottelu epäonnistui + +NAME_UNKNOWN = tuntematon nimi + +NON_PRODUCTION = tämä API-kutsu on kielletty tuotantoversiossa + +NO_TIME_SYNC = kello vielä synkronisoimatta + +ORDER_UNKNOWN = tuntematon resurssin tilaus-ID + +PUBLIC_KEY_NOT_FOUND = julkista avainta ei löytynyt + +REPOSITORY_ISSUE = tietovarantovirhe (repo) + +# This one is special in that caller expected to pass two additional strings, hence the two %s +TRANSACTION_INVALID = kelvoton transaktio: %s (%s) + +TRANSACTION_UNKNOWN = tuntematon transaktio + +TRANSFORMATION_ERROR = JSON:in muuntaminen transaktioksi epäonnistui + +UNAUTHORIZED = luvaton API-kutsu \ No newline at end of file diff --git a/src/main/resources/i18n/ApiError_it.properties b/src/main/resources/i18n/ApiError_it.properties new file mode 100644 index 00000000..27f93f63 --- /dev/null +++ b/src/main/resources/i18n/ApiError_it.properties @@ -0,0 +1,72 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# Keys are from api.ApiError enum +# Italian translation by Pabs 2021 + +# La modifica della lingua dell'UI è fatta nel file Settings.json +# +# "localeLang": "it", +# Si prega ricordare la virgola alla fine, se questo comando non è sull'ultima riga + +ADDRESS_UNKNOWN = indirizzo account sconosciuto + +BLOCKCHAIN_NEEDS_SYNC = blockchain deve prima sincronizzarsi + +# Blocks +BLOCK_UNKNOWN = blocco sconosciuto + +BTC_BALANCE_ISSUE = saldo Bitcoin insufficiente + +BTC_NETWORK_ISSUE = Bitcoin/ElectrumX problema di rete + +BTC_TOO_SOON = troppo presto per trasmettere transazione Bitcoin (tempo di blocco / tempo di blocco mediano) + +CANNOT_MINT = l'account non può coniare + +GROUP_UNKNOWN = gruppo sconosciuto + +INVALID_ADDRESS = indirizzo non valido + +# Assets +INVALID_ASSET_ID = identificazione risorsa non valida + +INVALID_CRITERIA = criteri di ricerca non validi + +INVALID_DATA = dati non validi + +INVALID_HEIGHT = altezza blocco non valida + +INVALID_NETWORK_ADDRESS = indirizzo di rete non valido + +INVALID_ORDER_ID = identificazione di ordine di risorsa non valida + +INVALID_PRIVATE_KEY = chiave privata non valida + +INVALID_PUBLIC_KEY = chiave pubblica non valida + +INVALID_REFERENCE = riferimento non valido + +# Validation +INVALID_SIGNATURE = firma non valida + +JSON = Impossibile analizzare il messaggio JSON + +NAME_UNKNOWN = nome sconosciuto + +NON_PRODUCTION = questa chiamata API non è consentita per i sistemi di produzione + +NO_TIME_SYNC = nessuna sincronizzazione dell'orologio ancora + +ORDER_UNKNOWN = identificazione di ordine di risorsa sconosciuta + +PUBLIC_KEY_NOT_FOUND = chiave pubblica non trovata + +REPOSITORY_ISSUE = errore del repositorio + +# This one is special in that caller expected to pass two additional strings, hence the two %s +TRANSACTION_INVALID = transazione non valida: %s (%s) + +TRANSACTION_UNKNOWN = transazione sconosciuta + +TRANSFORMATION_ERROR = non è stato possibile trasformare JSON in transazione + +UNAUTHORIZED = Chiamata API non autorizzata diff --git a/src/main/resources/i18n/SysTray_fi.properties b/src/main/resources/i18n/SysTray_fi.properties new file mode 100644 index 00000000..551b010e --- /dev/null +++ b/src/main/resources/i18n/SysTray_fi.properties @@ -0,0 +1,45 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# SysTray pop-up menu + +APPLYING_UPDATE_AND_RESTARTING = Automaattinen päivitys käynnissä, uudelleenkäynnistys seuraa... + +AUTO_UPDATE = Automaattinen päivitys + +BLOCK_HEIGHT = korkeus + +CHECK_TIME_ACCURACY = Tarkista ajan tarkkuus + +CONNECTING = Yhdistää + +CONNECTION = yhteys + +CONNECTIONS = yhteyttä + +CREATING_BACKUP_OF_DB_FILES = Luodaan varmuuskopio tietokannan tiedostoista... + +DB_BACKUP = Tietokannan varmuuskopio + +DB_CHECKPOINT = Tietokannan varmistuspiste + +EXIT = Pois + +MINTING_DISABLED = EI lyö rahaa + +MINTING_ENABLED = \u2714 Lyö rahaa + +# Nagging about lack of NTP time sync +NTP_NAG_CAPTION = Tietokoneen kello on epätarkka! + +NTP_NAG_TEXT_UNIX = Asennathan NTP-palvelun, jotta saat kellon tarkkuuden oikeaksi. + +NTP_NAG_TEXT_WINDOWS = Valitse "Kellon synkronisointi" valikosta korjataksesi. + +OPEN_UI = Avaa UI + +PERFORMING_DB_CHECKPOINT = Tallentaa kommittoidut tietokantamuutokset... + +SYNCHRONIZE_CLOCK = Synkronisoi kello + +SYNCHRONIZING_BLOCKCHAIN = Synkronisoi + +SYNCHRONIZING_CLOCK = Synkronisoi kelloa diff --git a/src/main/resources/i18n/SysTray_it.properties b/src/main/resources/i18n/SysTray_it.properties new file mode 100644 index 00000000..1d243958 --- /dev/null +++ b/src/main/resources/i18n/SysTray_it.properties @@ -0,0 +1,46 @@ +#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/) +# SysTray pop-up menu +# Italian translation by Pabs 2021 + +APPLYING_UPDATE_AND_RESTARTING = Applicando aggiornamento automatico e riavviando... + +AUTO_UPDATE = Aggiornamento automatico + +BLOCK_HEIGHT = altezza + +CHECK_TIME_ACCURACY = Controlla la precisione dell'ora + +CONNECTING = Collegando + +CONNECTION = connessione + +CONNECTIONS = connessioni + +CREATING_BACKUP_OF_DB_FILES = Creazione di backup dei file di database... + +DB_BACKUP = Backup del database + +DB_CHECKPOINT = Punto di controllo del database + +EXIT = Uscita + +MINTING_DISABLED = NON coniando + +MINTING_ENABLED = \u2714 Coniando + +# Nagging about lack of NTP time sync +NTP_NAG_CAPTION = L'orologio del computer è impreciso! + +NTP_NAG_TEXT_UNIX = Installare servizio NTP per ottenere un orologio preciso. + +NTP_NAG_TEXT_WINDOWS = Seleziona "Sincronizza orologio" dal menu per correggere. + +OPEN_UI = Apri UI + +PERFORMING_DB_CHECKPOINT = Salvataggio delle modifiche al database non salvate... + +SYNCHRONIZE_CLOCK = Sincronizza orologio + +SYNCHRONIZING_BLOCKCHAIN = Sincronizzando + +SYNCHRONIZING_CLOCK = Sincronizzando orologio diff --git a/src/main/resources/i18n/TransactionValidity_fi.properties b/src/main/resources/i18n/TransactionValidity_fi.properties new file mode 100644 index 00000000..2dc9abef --- /dev/null +++ b/src/main/resources/i18n/TransactionValidity_fi.properties @@ -0,0 +1,184 @@ + +ACCOUNT_ALREADY_EXISTS = tili on jo olemassa + +ACCOUNT_CANNOT_REWARD_SHARE = tili ei voi palkinto-jakaa + +ALREADY_GROUP_ADMIN = on jo ryhmän admin + +ALREADY_GROUP_MEMBER = on jo ryhmän jäsen + +ALREADY_VOTED_FOR_THAT_OPTION = on jo äänestänyt vaihtoehtoa + +ASSET_ALREADY_EXISTS = resurssi on jo olemassa + +ASSET_DOES_NOT_EXIST = resurssia ei ole olemassa + +ASSET_DOES_NOT_MATCH_AT = resurssi ei vastaa AT:n resurssia + +ASSET_NOT_SPENDABLE = resurssi ei ole kulutettavaa laatua + +AT_ALREADY_EXISTS = AT on jo olemassa + +AT_IS_FINISHED = AT on päättynyt + +AT_UNKNOWN = AT on tuntematon + +BANNED_FROM_GROUP = on evätty ryhmän jäsenyydestä + +BAN_EXISTS = eväys on jo olemassa + +BAN_UNKNOWN = tuntematon eväys + +BUYER_ALREADY_OWNER = ostaja on jo omistaja + +CHAT = CHATin transaktiot eivät koskaan ole kelvollisia sisällytettäväksi lohkoihin + +CLOCK_NOT_SYNCED = kello on synkronisoimatta + +DUPLICATE_OPTION = kahdennettu valinta + +GROUP_ALREADY_EXISTS = ryhmä on jo olemassa + +GROUP_APPROVAL_DECIDED = ryhmä-hyväksyminen jo päätetty + +GROUP_APPROVAL_NOT_REQUIRED = ryhmä-hyväksyminen tarpeeton + +GROUP_DOES_NOT_EXIST = ryhmää ei ole + +GROUP_ID_MISMATCH = ryhmän ID:n vastaavuusvirhe + +GROUP_OWNER_CANNOT_LEAVE = ryhmän omistaja ei voi jättää ryhmää + +HAVE_EQUALS_WANT = have-resurssi on sama kuin want-resurssi + +INCORRECT_NONCE = virheellinen PoW nonce + +INSUFFICIENT_FEE = riittämätön kulu + +INVALID_ADDRESS = kelvoton osoite + +INVALID_AMOUNT = kelvoton summa + +INVALID_ASSET_OWNER = kelvoton resurssin omistaja + +INVALID_AT_TRANSACTION = kelvoton AT-transaktio + +INVALID_AT_TYPE_LENGTH = kelvoton AT 'tyypin' pituus + +INVALID_CREATION_BYTES = kelvoton luodun tavumäärä + +INVALID_DATA_LENGTH = kelvoton datan pituus + +INVALID_DESCRIPTION_LENGTH = kelvoton kuvauksen pituus + +INVALID_GROUP_APPROVAL_THRESHOLD = kelvoton ryhmä-hyväksymisen alaraja + +INVALID_GROUP_BLOCK_DELAY = kelvoton ryhmä-hyväksymisen lohkon viive + +INVALID_GROUP_ID = kelvoton ryhmän ID + +INVALID_GROUP_OWNER = kelvoton ryhmän omistaja + +INVALID_LIFETIME = kelvoton elinaika + +INVALID_NAME_LENGTH = kelvoton nimen pituus + +INVALID_NAME_OWNER = kelvoton nimen omistaja + +INVALID_OPTIONS_COUNT = kelvoton valintojen lkm + +INVALID_OPTION_LENGTH = kelvoton valintojen pituus + +INVALID_ORDER_CREATOR = kelvoton tilauksen luoja + +INVALID_PAYMENTS_COUNT = kelvoton maksujen lkm + +INVALID_PUBLIC_KEY = kelvoton julkinen avain + +INVALID_QUANTITY = kelvoton määrä + +INVALID_REFERENCE = kelvoton viite + +INVALID_RETURN = kelvoton palautusarvo + +INVALID_REWARD_SHARE_PERCENT = kelvoton palkkiojaon prosenttiosuus + +INVALID_SELLER = kelvoton myyjä + +INVALID_TAGS_LENGTH = kelvoton 'tagin' pituus + +INVALID_TX_GROUP_ID = kelvoton transaktion ryhmä-ID + +INVALID_VALUE_LENGTH = kelvoton 'arvon' pituus + +INVITE_UNKNOWN = tuntematon ryhmän kutsu + +JOIN_REQUEST_EXISTS = ryhmään liittymispyyntö on jo olemassa + +MAXIMUM_REWARD_SHARES = tämän tilin suurin sallittu palkkiojaon lkm on saavutettu + +MISSING_CREATOR = luoja puuttuu + +MULTIPLE_NAMES_FORBIDDEN = yhdelle tilille sallitaan vain yksi rekisteröity nimi + +NAME_ALREADY_FOR_SALE = nimi on jo myynnissä + +NAME_ALREADY_REGISTERED = nimi on jo rekisteröity + +NAME_DOES_NOT_EXIST = nimeä ei ole + +NAME_NOT_FOR_SALE = nimi ei ole kaupan + +NAME_NOT_NORMALIZED = nimi ei ole Unicode 'normalisoitua' muotoa + +NEGATIVE_AMOUNT = kelvoton/negatiivinen summa + +NEGATIVE_FEE = kelvoton/negatiivinen kulu + +NEGATIVE_PRICE = kelvoton/negatiivinen hinta + +NOT_GROUP_ADMIN = tili ei ole ryhmän admin + +NOT_GROUP_MEMBER = tili ei ole ryhmän jäsen + +NOT_MINTING_ACCOUNT = tili ei voi lyödä rahaa + +NOT_YET_RELEASED = ominaisuutta ei ole vielä julkistettu + +NO_BALANCE = riittämätön saldo + +NO_BLOCKCHAIN_LOCK = solmun lohkoketju on juuri nyt varattuna + +NO_FLAG_PERMISSION = tilillä ei ole lupaa tuohon + +OK = OK + +ORDER_ALREADY_CLOSED = resurssin määräys kauppaan on jo suljettu + +ORDER_DOES_NOT_EXIST = resurssin määräystä kauppaan ei ole + +POLL_ALREADY_EXISTS = kysely on jo olemassa + +POLL_DOES_NOT_EXIST = kyselyä ei ole + +POLL_OPTION_DOES_NOT_EXIST = kyselyn tuota valintaa ei ole olemassa + +PUBLIC_KEY_UNKNOWN = tuntematon julkinen avain + +REWARD_SHARE_UNKNOWN = tuntematon palkkiojako + +SELF_SHARE_EXISTS = itse-jako (palkkiojako) on jo olemassa + +TIMESTAMP_TOO_NEW = aikaleima on liian tuore + +TIMESTAMP_TOO_OLD = aikaleima on liian vanha + +TOO_MANY_UNCONFIRMED = tilillä on liian monta vahvistamatonta transaktiota tekeillä + +TRANSACTION_ALREADY_CONFIRMED = transaktio on jo vahvistettu + +TRANSACTION_ALREADY_EXISTS = transaktio on jo olemassa + +TRANSACTION_UNKNOWN = tuntematon transaktio + +TX_GROUP_ID_MISMATCH = transaktion ryhmä-ID:n vastaavuusvirhe diff --git a/src/main/resources/i18n/TransactionValidity_it.properties b/src/main/resources/i18n/TransactionValidity_it.properties new file mode 100644 index 00000000..d97af856 --- /dev/null +++ b/src/main/resources/i18n/TransactionValidity_it.properties @@ -0,0 +1,185 @@ +# Italian translation by Pabs 2021 + +ACCOUNT_ALREADY_EXISTS = l'account gia esiste + +ACCOUNT_CANNOT_REWARD_SHARE = l'account non può fare la condivisione di ricompensa + +ALREADY_GROUP_ADMIN = è già amministratore del gruppo + +ALREADY_GROUP_MEMBER = è già membro del gruppo + +ALREADY_VOTED_FOR_THAT_OPTION = già votato per questa opzione + +ASSET_ALREADY_EXISTS = risorsa già esistente + +ASSET_DOES_NOT_EXIST = risorsa non esistente + +ASSET_DOES_NOT_MATCH_AT = l'asset non corrisponde all'asset di AT + +ASSET_NOT_SPENDABLE = la risorsa non è spendibile + +AT_ALREADY_EXISTS = AT gia esiste + +AT_IS_FINISHED = AT ha finito + +AT_UNKNOWN = AT sconosciuto + +BANNED_FROM_GROUP = divietato dal gruppo + +BAN_EXISTS = il divieto esiste già + +BAN_UNKNOWN = divieto sconosciuto + +BUYER_ALREADY_OWNER = l'acquirente è già proprietario + +CHAT = Le transazioni CHAT non sono mai valide per l'inclusione nei blocchi + +CLOCK_NOT_SYNCED = orologio non sincronizzato + +DUPLICATE_OPTION = opzione duplicata + +GROUP_ALREADY_EXISTS = gruppo già esistente + +GROUP_APPROVAL_DECIDED = approvazione di gruppo già decisa + +GROUP_APPROVAL_NOT_REQUIRED = approvazione di gruppo non richiesto + +GROUP_DOES_NOT_EXIST = gruppo non esiste + +GROUP_ID_MISMATCH = identificazione di gruppo non corrispondente + +GROUP_OWNER_CANNOT_LEAVE = il proprietario del gruppo non può lasciare il gruppo + +HAVE_EQUALS_WANT = la risorsa avere è uguale a la risorsa volere + +INCORRECT_NONCE = PoW nonce sbagliato + +INSUFFICIENT_FEE = tariffa insufficiente + +INVALID_ADDRESS = indirizzo non valido + +INVALID_AMOUNT = importo non valido + +INVALID_ASSET_OWNER = proprietario della risorsa non valido + +INVALID_AT_TRANSACTION = transazione AT non valida + +INVALID_AT_TYPE_LENGTH = lunghezza di "tipo" AT non valida + +INVALID_CREATION_BYTES = byte di creazione non validi + +INVALID_DATA_LENGTH = lunghezza di dati non valida + +INVALID_DESCRIPTION_LENGTH = lunghezza della descrizione non valida + +INVALID_GROUP_APPROVAL_THRESHOLD = soglia di approvazione del gruppo non valida + +INVALID_GROUP_BLOCK_DELAY = ritardo del blocco di approvazione del gruppo non valido + +INVALID_GROUP_ID = identificazione di gruppo non valida + +INVALID_GROUP_OWNER = proprietario di gruppo non valido + +INVALID_LIFETIME = durata della vita non valida + +INVALID_NAME_LENGTH = lunghezza del nome non valida + +INVALID_NAME_OWNER = proprietario del nome non valido + +INVALID_OPTIONS_COUNT = conteggio di opzioni non validi + +INVALID_OPTION_LENGTH = lunghezza di opzioni non valida + +INVALID_ORDER_CREATOR = creatore dell'ordine non valido + +INVALID_PAYMENTS_COUNT = conteggio pagamenti non validi + +INVALID_PUBLIC_KEY = chiave pubblica non valida + +INVALID_QUANTITY = quantità non valida + +INVALID_REFERENCE = riferimento non valido + +INVALID_RETURN = ritorno non valido + +INVALID_REWARD_SHARE_PERCENT = percentuale condivisione di ricompensa non valida + +INVALID_SELLER = venditore non valido + +INVALID_TAGS_LENGTH = lunghezza dei "tag" non valida + +INVALID_TX_GROUP_ID = identificazione di gruppo di transazioni non valida + +INVALID_VALUE_LENGTH = lunghezza "valore" non valida + +INVITE_UNKNOWN = invito di gruppo sconosciuto + +JOIN_REQUEST_EXISTS = la richiesta di iscrizione al gruppo già esiste + +MAXIMUM_REWARD_SHARES = numero massimo di condivisione di ricompensa raggiunto per l'account + +MISSING_CREATOR = creatore mancante + +MULTIPLE_NAMES_FORBIDDEN = è vietata la registrazione di multipli nomi per account + +NAME_ALREADY_FOR_SALE = nome già in vendita + +NAME_ALREADY_REGISTERED = nome già registrato + +NAME_DOES_NOT_EXIST = il nome non esiste + +NAME_NOT_FOR_SALE = il nome non è in vendita + +NAME_NOT_NORMALIZED = il nome non è in forma "normalizzata" Unicode + +NEGATIVE_AMOUNT = importo non valido / negativo + +NEGATIVE_FEE = tariffa non valida / negativa + +NEGATIVE_PRICE = prezzo non valido / negativo + +NOT_GROUP_ADMIN = l'account non è un amministratore di gruppo + +NOT_GROUP_MEMBER = l'account non è un membro del gruppo + +NOT_MINTING_ACCOUNT = l'account non può coniare + +NOT_YET_RELEASED = funzione non ancora rilasciata + +NO_BALANCE = equilibrio insufficiente + +NO_BLOCKCHAIN_LOCK = nodo di blockchain attualmente occupato + +NO_FLAG_PERMISSION = l'account non dispone di questa autorizzazione + +OK = OK + +ORDER_ALREADY_CLOSED = l'ordine di scambio di risorsa è già chiuso + +ORDER_DOES_NOT_EXIST = l'ordine di scambio di risorsa non esiste + +POLL_ALREADY_EXISTS = il sondaggio già esiste + +POLL_DOES_NOT_EXIST = il sondaggio non esiste + +POLL_OPTION_DOES_NOT_EXIST = le opzioni di sondaggio non esistono + +PUBLIC_KEY_UNKNOWN = chiave pubblica sconosciuta + +REWARD_SHARE_UNKNOWN = condivisione di ricompensa sconosciuta + +SELF_SHARE_EXISTS = condivisione di sé (condivisione di ricompensa) già esiste + +TIMESTAMP_TOO_NEW = timestamp troppo nuovo + +TIMESTAMP_TOO_OLD = timestamp troppo vecchio + +TOO_MANY_UNCONFIRMED = l'account ha troppe transazioni non confermate in sospeso + +TRANSACTION_ALREADY_CONFIRMED = la transazione è già confermata + +TRANSACTION_ALREADY_EXISTS = la transazione già esiste + +TRANSACTION_UNKNOWN = transazione sconosciuta + +TX_GROUP_ID_MISMATCH = identificazione di gruppo della transazione non corrisponde diff --git a/src/test/java/org/qortal/test/minting/RewardTests.java b/src/test/java/org/qortal/test/minting/RewardTests.java index 4d098f67..6c03662c 100644 --- a/src/test/java/org/qortal/test/minting/RewardTests.java +++ b/src/test/java/org/qortal/test/minting/RewardTests.java @@ -336,4 +336,457 @@ public class RewardTests extends Common { } } + /** Test rewards for level 1 and 2 accounts both pre and post the shareBinFix, including orphaning back through the feature trigger block */ + @Test + public void testLevel1And2Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share NOT online + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint a couple of blocks so that we are able to orphan them later + for (int i=0; i<2; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure that the levels are as we expect + assertEquals(1, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(2, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Ensure that only Alice is a founder + assertEquals(1, getFlags(repository, "alice")); + assertEquals(0, getFlags(repository, "bob")); + assertEquals(0, getFlags(repository, "chloe")); + assertEquals(0, getFlags(repository, "dilbert")); + + // Now that everyone is at level 1 or 2, we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are at the correct height and block reward value + assertEquals(6, (int) repository.getBlockRepository().getLastBlock().getHeight()); + assertEquals(10000000000L, blockReward); + + /* + * Alice, Chloe, and Dilbert are 'online'. Bob is offline. + * Chloe is level 1, Dilbert is level 2. + * One founder online (Alice, who is also level 1). + * No legacy QORA holders. + * + * Chloe and Dilbert should receive equal shares of the 5% block reward for Level 1 and 2 + * Alice should receive the remainder (95%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 1 and 2 to share the same reward (5%) + final int level1And2SharePercent = 5_00; // 5% + final long level1And2ShareAmount = (blockReward * level1And2SharePercent) / 100L / 100L; + final long expectedReward = level1And2ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderReward = blockReward - level1And2ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + assertEquals(500000000, level1And2ShareAmount); + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedReward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedReward); + + // Now orphan the latest block. This brings us to the threshold of the shareBinFix feature trigger. + BlockUtils.orphanBlocks(repository, 1); + assertEquals(5, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Ensure the latest post-fix block rewards have been subtracted and they have returned to their initial values + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance); + + // Orphan another block. This time, the block that was orphaned was prior to the shareBinFix feature trigger. + BlockUtils.orphanBlocks(repository, 1); + assertEquals(4, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Prior to the fix, the levels were incorrectly grouped + // Chloe should receive 100% of the level 1 reward, and Dilbert should receive 100% of the level 2+3 reward + final int level1SharePercent = 5_00; // 5% + final int level2And3SharePercent = 10_00; // 10% + final long level1ShareAmountBeforeFix = (blockReward * level1SharePercent) / 100L / 100L; + final long level2And3ShareAmountBeforeFix = (blockReward * level2And3SharePercent) / 100L / 100L; + final long expectedFounderRewardBeforeFix = blockReward - level1ShareAmountBeforeFix - level2And3ShareAmountBeforeFix; // Alice should receive the remainder + + // Validate the share amounts and balances + assertEquals(500000000, level1ShareAmountBeforeFix); + assertEquals(1000000000, level2And3ShareAmountBeforeFix); + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance-expectedFounderRewardBeforeFix); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance-level1ShareAmountBeforeFix); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance-level2And3ShareAmountBeforeFix); + + // Orphan the latest block one last time + BlockUtils.orphanBlocks(repository, 1); + assertEquals(3, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Validate balances + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance-(expectedFounderRewardBeforeFix*2)); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance-(level1ShareAmountBeforeFix*2)); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance-(level2And3ShareAmountBeforeFix*2)); + + } + } + + /** Test rewards for level 3 and 4 accounts */ + @Test + public void testLevel3And4Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share online + byte[] bobRewardSharePrivateKey = AccountUtils.rewardShare(repository, "bob", "bob", 0); + PrivateKeyAccount bobRewardShareAccount = new PrivateKeyAccount(repository, bobRewardSharePrivateKey); + mintingAndOnlineAccounts.add(bobRewardShareAccount); + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint enough blocks to bump testAccount levels to 3 and 4 + final int minterBlocksNeeded = cumulativeBlocksByLevel.get(4) - 20; // 20 blocks before level 4, so that the test accounts reach the correct levels + for (int bc = 0; bc < minterBlocksNeeded; ++bc) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure that the levels are as we expect + assertEquals(3, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(3, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(3, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(4, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Now that everyone is at level 3 or 4, we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are using the correct block reward value + assertEquals(100000000L, blockReward); + + /* + * Alice, Bob, Chloe, and Dilbert are 'online'. + * Bob and Chloe are level 3; Dilbert is level 4. + * One founder online (Alice, who is also level 3). + * No legacy QORA holders. + * + * Chloe, Bob and Dilbert should receive equal shares of the 10% block reward for level 3 and 4 + * Alice should receive the remainder (90%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 3 and 4 to share the same reward (10%) + final int level3And4SharePercent = 10_00; // 10% + final long level3And4ShareAmount = (blockReward * level3And4SharePercent) / 100L / 100L; + final long expectedReward = level3And4ShareAmount / 3; // The reward is split between Bob, Chloe, and Dilbert + final long expectedFounderReward = blockReward - level3And4ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance+expectedReward); + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedReward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedReward); + + } + } + + /** Test rewards for level 5 and 6 accounts */ + @Test + public void testLevel5And6Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share not initially online + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint enough blocks to bump testAccount levels to 5 and 6 + final int minterBlocksNeeded = cumulativeBlocksByLevel.get(6) - 20; // 20 blocks before level 6, so that the test accounts reach the correct levels + for (int bc = 0; bc < minterBlocksNeeded; ++bc) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Bob self-share now comes online + byte[] bobRewardSharePrivateKey = AccountUtils.rewardShare(repository, "bob", "bob", 0); + PrivateKeyAccount bobRewardShareAccount = new PrivateKeyAccount(repository, bobRewardSharePrivateKey); + mintingAndOnlineAccounts.add(bobRewardShareAccount); + + // Ensure that the levels are as we expect + assertEquals(5, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(5, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(6, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Now that everyone is at level 5 or 6 (except Bob who has only just started minting, so is at level 1), we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are using the correct block reward value + assertEquals(100000000L, blockReward); + + /* + * Alice, Bob, Chloe, and Dilbert are 'online'. + * Bob is level 1; Chloe is level 5; Dilbert is level 6. + * One founder online (Alice, who is also level 5). + * No legacy QORA holders. + * + * Chloe and Dilbert should receive equal shares of the 15% block reward for level 5 and 6 + * Bob should receive all of the level 1 and 2 reward (5%) + * Alice should receive the remainder (80%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 5 and 6 to share the same reward (15%) + final int level1And2SharePercent = 5_00; // 5% + final int level5And6SharePercent = 15_00; // 10% + final long level1And2ShareAmount = (blockReward * level1And2SharePercent) / 100L / 100L; + final long level5And6ShareAmount = (blockReward * level5And6SharePercent) / 100L / 100L; + final long expectedLevel1And2Reward = level1And2ShareAmount; // The reward is given entirely to Bob + final long expectedLevel5And6Reward = level5And6ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderReward = blockReward - level1And2ShareAmount - level5And6ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance+expectedLevel1And2Reward); + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedLevel5And6Reward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedLevel5And6Reward); + + } + } + + /** Test rewards for level 7 and 8 accounts */ + @Test + public void testLevel7And8Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share NOT online + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint enough blocks to bump testAccount levels to 7 and 8 + final int minterBlocksNeeded = cumulativeBlocksByLevel.get(8) - 20; // 20 blocks before level 8, so that the test accounts reach the correct levels + for (int bc = 0; bc < minterBlocksNeeded; ++bc) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure that the levels are as we expect + assertEquals(7, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(7, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(8, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Now that everyone is at level 7 or 8 (except Bob who has only just started minting, so is at level 1), we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are using the correct block reward value + assertEquals(100000000L, blockReward); + + /* + * Alice, Chloe, and Dilbert are 'online'. + * Chloe is level 7; Dilbert is level 8. + * One founder online (Alice, who is also level 7). + * No legacy QORA holders. + * + * Chloe and Dilbert should receive equal shares of the 20% block reward for level 7 and 8 + * Alice should receive the remainder (80%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 7 and 8 to share the same reward (20%) + final int level7And8SharePercent = 20_00; // 20% + final long level7And8ShareAmount = (blockReward * level7And8SharePercent) / 100L / 100L; + final long expectedLevel7And8Reward = level7And8ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderReward = blockReward - level7And8ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedLevel7And8Reward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedLevel7And8Reward); + + } + } + + /** Test rewards for level 9 and 10 accounts */ + @Test + public void testLevel9And10Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share not initially online + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint enough blocks to bump testAccount levels to 9 and 10 + final int minterBlocksNeeded = cumulativeBlocksByLevel.get(10) - 20; // 20 blocks before level 10, so that the test accounts reach the correct levels + for (int bc = 0; bc < minterBlocksNeeded; ++bc) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Bob self-share now comes online + byte[] bobRewardSharePrivateKey = AccountUtils.rewardShare(repository, "bob", "bob", 0); + PrivateKeyAccount bobRewardShareAccount = new PrivateKeyAccount(repository, bobRewardSharePrivateKey); + mintingAndOnlineAccounts.add(bobRewardShareAccount); + + // Ensure that the levels are as we expect + assertEquals(9, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(9, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(10, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Now that everyone is at level 7 or 8 (except Bob who has only just started minting, so is at level 1), we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are using the correct block reward value + assertEquals(100000000L, blockReward); + + /* + * Alice, Bob, Chloe, and Dilbert are 'online'. + * Bob is level 1; Chloe is level 9; Dilbert is level 10. + * One founder online (Alice, who is also level 9). + * No legacy QORA holders. + * + * Chloe and Dilbert should receive equal shares of the 25% block reward for level 9 and 10 + * Bob should receive all of the level 1 and 2 reward (5%) + * Alice should receive the remainder (70%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 9 and 10 to share the same reward (25%) + final int level1And2SharePercent = 5_00; // 5% + final int level9And10SharePercent = 25_00; // 25% + final long level1And2ShareAmount = (blockReward * level1And2SharePercent) / 100L / 100L; + final long level9And10ShareAmount = (blockReward * level9And10SharePercent) / 100L / 100L; + final long expectedLevel1And2Reward = level1And2ShareAmount; // The reward is given entirely to Bob + final long expectedLevel9And10Reward = level9And10ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderReward = blockReward - level1And2ShareAmount - level9And10ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance+expectedLevel1And2Reward); + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedLevel9And10Reward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedLevel9And10Reward); + + } + } + + + private int getFlags(Repository repository, String name) throws DataException { + TestAccount testAccount = Common.getTestAccount(repository, name); + return repository.getAccountRepository().getAccount(testAccount.getAddress()).getFlags(); + } + } \ No newline at end of file diff --git a/src/test/resources/test-chain-v2-founder-rewards.json b/src/test/resources/test-chain-v2-founder-rewards.json index 6ffe946a..2b96da55 100644 --- a/src/test/resources/test-chain-v2-founder-rewards.json +++ b/src/test/resources/test-chain-v2-founder-rewards.json @@ -46,6 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, "calcChainWeightTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-leftover-reward.json b/src/test/resources/test-chain-v2-leftover-reward.json index 2cf6f2ab..3ff0c8e7 100644 --- a/src/test/resources/test-chain-v2-leftover-reward.json +++ b/src/test/resources/test-chain-v2-leftover-reward.json @@ -46,6 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, "calcChainWeightTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-minting.json b/src/test/resources/test-chain-v2-minting.json index 4370f52b..94014868 100644 --- a/src/test/resources/test-chain-v2-minting.json +++ b/src/test/resources/test-chain-v2-minting.json @@ -46,6 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, "calcChainWeightTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-qora-holder-extremes.json b/src/test/resources/test-chain-v2-qora-holder-extremes.json index 1c0f0a7b..308461c1 100644 --- a/src/test/resources/test-chain-v2-qora-holder-extremes.json +++ b/src/test/resources/test-chain-v2-qora-holder-extremes.json @@ -46,6 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, "calcChainWeightTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-qora-holder.json b/src/test/resources/test-chain-v2-qora-holder.json index ddb3cac9..99adf1be 100644 --- a/src/test/resources/test-chain-v2-qora-holder.json +++ b/src/test/resources/test-chain-v2-qora-holder.json @@ -46,6 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, "calcChainWeightTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-reward-levels.json b/src/test/resources/test-chain-v2-reward-levels.json new file mode 100644 index 00000000..a078119a --- /dev/null +++ b/src/test/resources/test-chain-v2-reward-levels.json @@ -0,0 +1,75 @@ +{ + "isTestChain": true, + "blockTimestampMargin": 500, + "transactionExpiryPeriod": 86400000, + "maxBlockSize": 2097152, + "maxBytesPerUnitFee": 1024, + "unitFee": "0.1", + "requireGroupForApproval": false, + "minAccountLevelToRewardShare": 5, + "maxRewardSharesPerMintingAccount": 20, + "founderEffectiveMintingLevel": 10, + "onlineAccountSignaturesMinLifetime": 3600000, + "onlineAccountSignaturesMaxLifetime": 86400000, + "rewardsByHeight": [ + { "height": 1, "reward": 100 }, + { "height": 11, "reward": 10 }, + { "height": 21, "reward": 1 } + ], + "sharesByLevel": [ + { "levels": [ 1, 2 ], "share": 0.05 }, + { "levels": [ 3, 4 ], "share": 0.10 }, + { "levels": [ 5, 6 ], "share": 0.15 }, + { "levels": [ 7, 8 ], "share": 0.20 }, + { "levels": [ 9, 10 ], "share": 0.25 } + ], + "qoraHoldersShare": 0.20, + "qoraPerQortReward": 250, + "blocksNeededByLevel": [ 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 ], + "blockTimingsByHeight": [ + { "height": 1, "target": 60000, "deviation": 30000, "power": 0.2 } + ], + "ciyamAtSettings": { + "feePerStep": "0.0001", + "maxStepsPerRound": 500, + "stepsPerFunctionCall": 10, + "minutesPerBlock": 1 + }, + "featureTriggers": { + "messageHeight": 0, + "atHeight": 0, + "assetsTimestamp": 0, + "votingTimestamp": 0, + "arbitraryTimestamp": 0, + "powfixTimestamp": 0, + "qortalTimestamp": 0, + "newAssetPricingTimestamp": 0, + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 6, + "calcChainWeightTimestamp": 0 + }, + "genesisInfo": { + "version": 4, + "timestamp": 0, + "transactions": [ + { "type": "ISSUE_ASSET", "assetName": "QORT", "description": "QORT native coin", "data": "", "quantity": 0, "isDivisible": true, "fee": 0 }, + { "type": "ISSUE_ASSET", "assetName": "Legacy-QORA", "description": "Representative legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true }, + { "type": "ISSUE_ASSET", "assetName": "QORT-from-QORA", "description": "QORT gained from holding legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true }, + + { "type": "GENESIS", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "amount": "1000000000" }, + { "type": "GENESIS", "recipient": "QixPbJUwsaHsVEofJdozU9zgVqkK6aYhrK", "amount": "1000000" }, + { "type": "GENESIS", "recipient": "QaUpHNhT3Ygx6avRiKobuLdusppR5biXjL", "amount": "1000000" }, + { "type": "GENESIS", "recipient": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "amount": "1000000" }, + + { "type": "ACCOUNT_FLAGS", "target": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "andMask": -1, "orMask": 1, "xorMask": 0 }, + { "type": "REWARD_SHARE", "minterPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "rewardSharePublicKey": "7PpfnvLSG7y4HPh8hE7KoqAjLCkv7Ui6xw4mKAkbZtox", "sharePercent": 100 }, + + { "type": "ACCOUNT_LEVEL", "target": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "level": 1 }, + { "type": "ACCOUNT_LEVEL", "target": "QixPbJUwsaHsVEofJdozU9zgVqkK6aYhrK", "level": 1 }, + { "type": "ACCOUNT_LEVEL", "target": "QaUpHNhT3Ygx6avRiKobuLdusppR5biXjL", "level": 1 }, + { "type": "ACCOUNT_LEVEL", "target": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "level": 2 } + ] + } +} diff --git a/src/test/resources/test-chain-v2-reward-scaling.json b/src/test/resources/test-chain-v2-reward-scaling.json index c588bc9f..e0faeec2 100644 --- a/src/test/resources/test-chain-v2-reward-scaling.json +++ b/src/test/resources/test-chain-v2-reward-scaling.json @@ -46,6 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, "calcChainWeightTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index e4f20209..e7347246 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -46,6 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 999999, "calcChainWeightTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-settings-v2-reward-levels.json b/src/test/resources/test-settings-v2-reward-levels.json new file mode 100644 index 00000000..1c6862ad --- /dev/null +++ b/src/test/resources/test-settings-v2-reward-levels.json @@ -0,0 +1,7 @@ +{ + "restrictedApi": false, + "blockchainConfig": "src/test/resources/test-chain-v2-reward-levels.json", + "wipeUnconfirmedOnStart": false, + "testNtpOffset": 0, + "minPeers": 0 +} diff --git a/stop.sh b/stop.sh index 2f26bc1f..90cc0a7a 100755 --- a/stop.sh +++ b/stop.sh @@ -21,21 +21,38 @@ fi read pid 2>/dev/null /dev/null 2>&1; then - echo "Qortal node responded and should be shutting down" - if [ "${is_pid_valid}" -eq 0 ]; then - echo -n "Monitoring for Qortal node to end" - while s=`ps -p $pid -o stat=` && [[ "$s" && "$s" != 'Z' ]]; do - echo -n . - sleep 1 - done - echo - echo "${green}Qortal ended gracefully${normal}" - rm -f run.pid +# Swap out the API port if the --testnet (or -t) argument is specified +api_port=12391 +if [[ "$@" = *"--testnet"* ]] || [[ "$@" = *"-t"* ]]; then + api_port=62391 +fi + +# Ensure curl is installed +curl_path=$(which curl) + +if [[ -f $curl_path ]]; then + + echo 'Calling GET /admin/stop on local Qortal node' + if curl --url "http://localhost:${api_port}/admin/stop" 1>/dev/null 2>&1; then + echo "Qortal node responded and should be shutting down" + + if [ "${is_pid_valid}" -eq 0 ]; then + echo -n "Monitoring for Qortal node to end" + while s=`ps -p $pid -o stat=` && [[ "$s" && "$s" != 'Z' ]]; do + echo -n . + sleep 1 + done + echo + echo "${green}Qortal ended gracefully${normal}" + rm -f run.pid + fi + exit 0 + else + echo "${red}No response from Qortal node - not running on port ${api_port}?${normal}" + exit 1 fi - exit 0 + else - echo "${red}No response from Qortal node - not running?${normal}" + echo "${red}curl is not installed or in the path${normal}" exit 1 fi diff --git a/tools/publish-auto-update.pl b/tools/publish-auto-update.pl index 3493f964..ad43b2f4 100755 --- a/tools/publish-auto-update.pl +++ b/tools/publish-auto-update.pl @@ -57,9 +57,11 @@ $timestamp *= 1000; # Convert to milliseconds # locate sha256 utility my $SHA256 = `which sha256sum || which sha256`; +chomp $SHA256; +die("Can't find sha256sum or sha256\n") unless length($SHA256) > 0; # SHA256 of actual update file -my $sha256 = `git show auto-update-${commit_hash}:${project}.update | ${SHA256}`; +my $sha256 = `git show auto-update-${commit_hash}:${project}.update | ${SHA256} | head -c 64`; die("Can't calculate SHA256 of ${project}.update\n") unless $sha256 =~ m/(\S{64})/; chomp $sha256;