From 641a6580593b761a007ffa3e2e0eb5d2ebb47350 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Apr 2021 17:49:04 +0100 Subject: [PATCH 01/15] Added /blocks/byheight/{height}/mintinginfo API, which returns info on the minter level, key distance, and block timings. --- .../qortal/api/model/BlockMintingInfo.java | 22 +++++++ .../qortal/api/resource/BlocksResource.java | 58 +++++++++++++++++++ src/main/java/org/qortal/block/Block.java | 2 +- 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/qortal/api/model/BlockMintingInfo.java diff --git a/src/main/java/org/qortal/api/model/BlockMintingInfo.java b/src/main/java/org/qortal/api/model/BlockMintingInfo.java new file mode 100644 index 00000000..e71c918b --- /dev/null +++ b/src/main/java/org/qortal/api/model/BlockMintingInfo.java @@ -0,0 +1,22 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.math.BigDecimal; +import java.math.BigInteger; + +@XmlAccessorType(XmlAccessType.FIELD) +public class BlockMintingInfo { + + public byte[] minterPublicKey; + public int minterLevel; + public BigDecimal maxDistance; + public BigInteger keyDistance; + public double keyDistanceRatio; + public long timestamp; + public long timeDelta; + + public BlockMintingInfo() { + } + +} diff --git a/src/main/java/org/qortal/api/resource/BlocksResource.java b/src/main/java/org/qortal/api/resource/BlocksResource.java index 30cc477e..1b160b91 100644 --- a/src/main/java/org/qortal/api/resource/BlocksResource.java +++ b/src/main/java/org/qortal/api/resource/BlocksResource.java @@ -8,6 +8,9 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; import java.util.ArrayList; import java.util.List; @@ -20,10 +23,13 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import org.qortal.account.Account; import org.qortal.api.ApiError; import org.qortal.api.ApiErrors; import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.model.BlockMintingInfo; import org.qortal.api.model.BlockSignerSummary; +import org.qortal.block.Block; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountData; import org.qortal.data.block.BlockData; @@ -328,6 +334,58 @@ public class BlocksResource { } } + @GET + @Path("/byheight/{height}/mintinginfo") + @Operation( + summary = "Fetch block minter info using block height", + description = "Returns the minter info for the block with given height", + responses = { + @ApiResponse( + description = "the block", + content = @Content( + schema = @Schema( + implementation = BlockData.class + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE + }) + public BlockMintingInfo getBlockMintingInfoByHeight(@PathParam("height") int height) { + try (final Repository repository = RepositoryManager.getRepository()) { + BlockData blockData = repository.getBlockRepository().fromHeight(height); + if (blockData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + + Block block = new Block(repository, blockData); + BlockData parentBlockData = repository.getBlockRepository().fromSignature(blockData.getReference()); + int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey()); + if (minterLevel == 0) + // This may be unavailable when requesting a trimmed block + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + BigInteger distance = block.calcKeyDistance(parentBlockData.getHeight(), parentBlockData.getSignature(), blockData.getMinterPublicKey(), minterLevel); + double ratio = new BigDecimal(distance).divide(new BigDecimal(block.MAX_DISTANCE), 40, RoundingMode.DOWN).doubleValue(); + long timestamp = block.calcTimestamp(parentBlockData, blockData.getMinterPublicKey(), minterLevel); + long timeDelta = timestamp - parentBlockData.getTimestamp(); + + BlockMintingInfo blockMintingInfo = new BlockMintingInfo(); + blockMintingInfo.minterPublicKey = blockData.getMinterPublicKey(); + blockMintingInfo.minterLevel = minterLevel; + blockMintingInfo.maxDistance = new BigDecimal(block.MAX_DISTANCE); + blockMintingInfo.keyDistance = distance; + blockMintingInfo.keyDistanceRatio = ratio; + blockMintingInfo.timestamp = timestamp; + blockMintingInfo.timeDelta = timeDelta; + + return blockMintingInfo; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + @GET @Path("/timestamp/{timestamp}") @Operation( diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 8551e4e7..74fe059c 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -225,7 +225,7 @@ public class Block { // Other useful constants - private static final BigInteger MAX_DISTANCE; + public static final BigInteger MAX_DISTANCE; static { byte[] maxValue = new byte[Transformer.PUBLIC_KEY_LENGTH]; Arrays.fill(maxValue, (byte) 0xFF); From a1a1b8e94a1236b29f8b59625872077bb77589f8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Apr 2021 17:57:28 +0100 Subject: [PATCH 02/15] Added tools/block-timings-sh which can be used to test out new block timings (specified in blockchain.json). The script will fetch a set of blocks and then backtest the specified blockTimings settings (target, deviation, and power) against those real life blocks. This allows configurations to be fine tuned to tighten up block times, and to adjust the timestamp variance between levels. Usage: block-timings.sh [target] [deviation] [power] startheight: a block height, preferably within the untrimmed range, to avoid data gaps count: the number of blocks to request and analyse after the start height. Default: 100 target: the target block time in milliseconds. Originates from blockchain.json. Default: 60000 deviation: the allowed block time deviation in milliseconds. Originates from blockchain.json. Default: 30000 power: used when transforming key distance to a time offset. Originates from blockchain.json. Default: 0.2 --- tools/block-timings.sh | 147 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100755 tools/block-timings.sh diff --git a/tools/block-timings.sh b/tools/block-timings.sh new file mode 100755 index 00000000..43f2f466 --- /dev/null +++ b/tools/block-timings.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash + +start_height=$1 +count=$2 + +if [ -z "${start_height}" ]; then + echo + echo "Error: missing start height." + echo + echo "Usage:" + echo "block-timings.sh [target] [deviation] [power]" + echo + echo "startheight: a block height, preferably within the untrimmed range, to avoid data gaps" + echo "count: the number of blocks to request and analyse after the start height. Default: 100" + echo "target: the target block time in milliseconds. Originates from blockchain.json. Default: 60000" + echo "deviation: the allowed block time deviation in milliseconds. Originates from blockchain.json. Default: 30000" + echo "power: used when transforming key distance to a time offset. Originates from blockchain.json. Default: 0.2" + echo + exit +fi + +target=$3 +deviation=$4 +power=$5 + +count=${count:=100} +target=${target:=60000} +deviation=${deviation:=30000} +power=${power:=0.2} + +finish_height=$((start_height + count - 1)) +height=$start_height + +echo "Settings:" +echo "Target time offset: ${target}" +echo "Deviation: ${deviation}" +echo "Power transform: ${power}" +echo + +function calculate_time_offset { + local key_distance_ratio=$1 + local transformed=$( echo "" | awk "END {print ${key_distance_ratio} ^ ${power}}") + local time_offset=$(echo "${deviation}*2*${transformed}" | bc) + time_offset=${time_offset%.*} + echo $time_offset +} + + +function fetch_and_process_blocks { + + echo "Fetching blocks from height ${start_height} to ${finish_height}..." + echo + + total_time_offset=0 + errors=0 + + while [ "${height}" -le "${finish_height}" ]; do + block_minting_info=$(curl -s "http://localhost:12391/blocks/byheight/${height}/mintinginfo") + error=$(echo "${block_minting_info}" | jq -r .error) + if [ "${error}" != "null" ]; then + echo "Error fetching minting info for block ${height}" + echo + errors=$((errors+1)) + height=$((height+1)) + continue; + fi + + # Parse minting info + minter_level=$(echo "${block_minting_info}" | jq -r .minterLevel) + key_distance_ratio=$(echo "${block_minting_info}" | jq -r .keyDistanceRatio) + time_delta=$(echo "${block_minting_info}" | jq -r .timeDelta) + + time_offset=$(calculate_time_offset "${key_distance_ratio}") + block_time=$((target-deviation+time_offset)) + + echo "=== BLOCK ${height} ===" + echo "Minter level: ${minter_level}" + echo "Key distance ratio: ${key_distance_ratio}" + echo "Time offset: ${time_offset}" + echo "Block time (real): ${time_delta}" + echo "Block time (calculated): ${block_time}" + + if [ "${time_delta}" -ne "${block_time}" ]; then + echo "WARNING: Block time mismatch. This is to be expected when using custom settings." + fi + echo + + total_time_offset=$((total_time_offset+block_time)) + + height=$((height+1)) + done + + adjusted_count=$((count-errors)) + if [ "${adjusted_count}" -eq 0 ]; then + echo "No blocks were retrieved." + echo + exit; + fi + + mean_time_offset=$((total_time_offset/adjusted_count)) + time_offset_diff=$((mean_time_offset-target)) + + echo "===================" + echo "===== SUMMARY =====" + echo "===================" + echo "Total blocks retrieved: ${adjusted_count}" + echo "Total blocks failed: ${errors}" + echo "Mean time offset: ${mean_time_offset}ms" + echo "Target time offset: ${target}ms" + echo "Difference from target: ${time_offset_diff}ms" + echo + +} + +function estimate_key_distance_ratio_for_level { + local level=$1 + local example_key_distance="0.5" + echo "(${example_key_distance}/${level})" +} + +function estimate_block_timestamps { + min_block_time=9999999 + max_block_time=0 + + echo "===== BLOCK TIME ESTIMATES =====" + + for level in {1..10}; do + example_key_distance_ratio=$(estimate_key_distance_ratio_for_level "${level}") + time_offset=$(calculate_time_offset "${example_key_distance_ratio}") + block_time=$((target-deviation+time_offset)) + + if [ "${block_time}" -gt "${max_block_time}" ]; then + max_block_time=${block_time} + fi + if [ "${block_time}" -lt "${min_block_time}" ]; then + min_block_time=${block_time} + fi + + echo "Level: ${level}, time offset: ${time_offset}, block time: ${block_time}" + done + block_time_range=$((max_block_time-min_block_time)) + echo "Range: ${block_time_range}" + echo +} + +fetch_and_process_blocks +estimate_block_timestamps From 78cac7f0e64bb78b472aad339d1c060517f55835 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Apr 2021 18:12:09 +0100 Subject: [PATCH 03/15] Updated usage info to reflect the fact that the "count" parameter is optional. Usage: block-timings.sh [count] [target] [deviation] [power] --- tools/block-timings.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/block-timings.sh b/tools/block-timings.sh index 43f2f466..514168dd 100755 --- a/tools/block-timings.sh +++ b/tools/block-timings.sh @@ -8,7 +8,7 @@ if [ -z "${start_height}" ]; then echo "Error: missing start height." echo echo "Usage:" - echo "block-timings.sh [target] [deviation] [power]" + echo "block-timings.sh [count] [target] [deviation] [power]" echo echo "startheight: a block height, preferably within the untrimmed range, to avoid data gaps" echo "count: the number of blocks to request and analyse after the start height. Default: 100" From 45efe7cd5689dde7e010de4187b59da549f3ebfe Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Apr 2021 18:24:33 +0100 Subject: [PATCH 04/15] Slight reordering of vars. --- tools/block-timings.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/block-timings.sh b/tools/block-timings.sh index 514168dd..5ca4f08a 100755 --- a/tools/block-timings.sh +++ b/tools/block-timings.sh @@ -2,6 +2,9 @@ start_height=$1 count=$2 +target=$3 +deviation=$4 +power=$5 if [ -z "${start_height}" ]; then echo @@ -19,10 +22,6 @@ if [ -z "${start_height}" ]; then exit fi -target=$3 -deviation=$4 -power=$5 - count=${count:=100} target=${target:=60000} deviation=${deviation:=30000} From 475802afbc3dd5c6fb2854852e269eda62870f70 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 May 2021 08:25:24 +0100 Subject: [PATCH 05/15] Fixed divide by zero exception. Block.calcKeyDistance() cannot be called on some trimmed blocks, because the minter level is unable to be inferred in some cases. This generally hasn't been an issue, but the new Block.logDebugInfo() method is invoking it for all blocks. For now I am adding defensiveness to the debug method, but longer term we might want to add defensiveness to Block.calcKeyDistance() itself, if we ever encounter this issue again. I will leave it alone for now, to reduce risk. --- src/main/java/org/qortal/block/Block.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index fda06d73..25959e1f 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -2009,7 +2009,7 @@ public class Block { LOGGER.debug(String.format("Online accounts: %d", this.getBlockData().getOnlineAccountsCount())); BlockSummaryData blockSummaryData = new BlockSummaryData(this.getBlockData()); - if (this.getParent() == null || this.getParent().getSignature() == null || blockSummaryData == null) + if (this.getParent() == null || this.getParent().getSignature() == null || blockSummaryData == null || minterLevel == 0) return; blockSummaryData.setMinterLevel(minterLevel); From f4520e27521cd0a70081929ee6d05a59a8886d93 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 May 2021 09:00:53 +0100 Subject: [PATCH 06/15] Skip Block.logDebugInfo() altogether if the log level is more specific than DEBUG, to avoid wasting resources. --- src/main/java/org/qortal/block/Block.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 25959e1f..e2a15911 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1998,6 +1998,10 @@ public class Block { private void logDebugInfo() { try { + // Avoid calculations if possible. We have to check against INFO here, since Level.isMoreSpecificThan() confusingly uses <= rather than just < + if (LOGGER.getLevel().isMoreSpecificThan(Level.INFO)) + return; + if (this.repository == null || this.getMinter() == null || this.getBlockData() == null) return; From 8260cec7138331d7bc30a5b75e3ae2865f30ec3f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 May 2021 15:56:15 +0100 Subject: [PATCH 07/15] Added "maximumCount" parameter to HSQLDBATRepository.getMatchingFinalATStatesQuorum() and use it to limit the number of ATs being returned in the query. Initially set to 10 when used by the /crosschain/price/{blockchain} API, so that the price is based on the last 10 trades rather than every trade that has ever taken place. --- .../java/org/qortal/api/resource/CrossChainResource.java | 3 ++- src/main/java/org/qortal/repository/ATRepository.java | 2 +- .../org/qortal/repository/hsqldb/HSQLDBATRepository.java | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/CrossChainResource.java b/src/main/java/org/qortal/api/resource/CrossChainResource.java index 7a6c2c96..d1692b71 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainResource.java @@ -262,6 +262,7 @@ public class CrossChainResource { // We want both a minimum of 5 trades and enough trades to span at least 4 hours int minimumCount = 5; + int maximumCount = 10; long minimumPeriod = 4 * 60 * 60 * 1000L; // ms Boolean isFinished = Boolean.TRUE; @@ -276,7 +277,7 @@ public class CrossChainResource { ACCT acct = acctInfo.getValue().get(); List atStates = repository.getATRepository().getMatchingFinalATStatesQuorum(codeHash, - isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumCount, minimumPeriod); + isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumCount, maximumCount, minimumPeriod); for (ATStateData atState : atStates) { CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState); diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 0854a21c..5516ac28 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -98,7 +98,7 @@ public interface ATRepository { */ public List getMatchingFinalATStatesQuorum(byte[] codeHash, Boolean isFinished, Integer dataByteOffset, Long expectedValue, - int minimumCount, long minimumPeriod) throws DataException; + int minimumCount, int maximumCount, long minimumPeriod) throws DataException; /** * Returns all ATStateData for a given block height. diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index f82e4e62..8193c5d2 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -454,7 +454,7 @@ public class HSQLDBATRepository implements ATRepository { @Override public List getMatchingFinalATStatesQuorum(byte[] codeHash, Boolean isFinished, Integer dataByteOffset, Long expectedValue, - int minimumCount, long minimumPeriod) throws DataException { + int minimumCount, int maximumCount, long minimumPeriod) throws DataException { // We need most recent entry first so we can use its timestamp to slice further results List mostRecentStates = this.getMatchingFinalATStates(codeHash, isFinished, dataByteOffset, expectedValue, null, @@ -510,7 +510,8 @@ public class HSQLDBATRepository implements ATRepository { bindParams.add(minimumHeight); bindParams.add(minimumCount); - sql.append("ORDER BY FinalATStates.height DESC"); + sql.append("ORDER BY FinalATStates.height DESC LIMIT ?"); + bindParams.add(maximumCount); List atStates = new ArrayList<>(); From c4cbb64643e81c3d01bf51d59b73fa901bdba847 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 May 2021 17:38:07 +0100 Subject: [PATCH 08/15] Added "minPeerVersion" setting, and avoid syncing with peers on lower versions. --- .../org/qortal/controller/Controller.java | 8 +++++ .../java/org/qortal/network/Handshake.java | 5 +-- src/main/java/org/qortal/network/Peer.java | 35 +++++++++++++++++++ .../java/org/qortal/settings/Settings.java | 4 +++ 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index a7d028bc..c1386f53 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -623,6 +623,11 @@ public class Controller extends Thread { return peerChainTipData == null || peerChainTipData.getLastBlockSignature() == null || inferiorChainTips.contains(new ByteArray(peerChainTipData.getLastBlockSignature())); }; + public static final Predicate hasOldVersion = peer -> { + final String minPeerVersion = Settings.getInstance().getMinPeerVersion(); + return peer.isAtLeastVersion(minPeerVersion) == false; + }; + private void potentiallySynchronize() throws InterruptedException { // Already synchronizing via another thread? if (this.isSynchronizing) @@ -656,6 +661,9 @@ public class Controller extends Thread { // Disregard peers that are on the same block as last sync attempt and we didn't like their chain peers.removeIf(hasInferiorChainTip); + // Disregard peers that are on an old version + peers.removeIf(hasOldVersion); + final int peersBeforeComparison = peers.size(); // Request recent block summaries from the remaining peers, and locate our common block with each diff --git a/src/main/java/org/qortal/network/Handshake.java b/src/main/java/org/qortal/network/Handshake.java index 3c033e88..ed6d02db 100644 --- a/src/main/java/org/qortal/network/Handshake.java +++ b/src/main/java/org/qortal/network/Handshake.java @@ -4,7 +4,6 @@ import java.util.Arrays; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -51,7 +50,7 @@ public enum Handshake { String versionString = helloMessage.getVersionString(); - Matcher matcher = VERSION_PATTERN.matcher(versionString); + Matcher matcher = peer.VERSION_PATTERN.matcher(versionString); if (!matcher.lookingAt()) { LOGGER.debug(() -> String.format("Peer %s sent invalid HELLO version string '%s'", peer, versionString)); return null; @@ -244,8 +243,6 @@ public enum Handshake { /** Maximum allowed difference between peer's reported timestamp and when they connected, in milliseconds. */ private static final long MAX_TIMESTAMP_DELTA = 30 * 1000L; // ms - private static final Pattern VERSION_PATTERN = Pattern.compile(Controller.VERSION_PREFIX + "(\\d{1,3})\\.(\\d{1,5})\\.(\\d{1,5})"); - private static final long PEER_VERSION_131 = 0x0100030001L; private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index 08db0dd9..ffc90dc7 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -20,9 +20,12 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; import org.qortal.data.block.CommonBlockData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; @@ -87,6 +90,9 @@ public class Peer { byte[] ourChallenge; + // Versioning + public static final Pattern VERSION_PATTERN = Pattern.compile(Controller.VERSION_PREFIX + "(\\d{1,3})\\.(\\d{1,5})\\.(\\d{1,5})"); + // Peer info private final Object peerInfoLock = new Object(); @@ -634,6 +640,35 @@ public class Peer { } + // Minimum version + + public boolean isAtLeastVersion(String minVersionString) { + if (minVersionString == null) + return false; + + // Add the version prefix + minVersionString = Controller.VERSION_PREFIX + minVersionString; + + Matcher matcher = VERSION_PATTERN.matcher(minVersionString); + if (!matcher.lookingAt()) + return false; + + // We're expecting 3 positive shorts, so we can convert 1.2.3 into 0x0100020003 + long minVersion = 0; + for (int g = 1; g <= 3; ++g) { + long value = Long.parseLong(matcher.group(g)); + + if (value < 0 || value > Short.MAX_VALUE) + return false; + + minVersion <<= 16; + minVersion |= value; + } + + return this.getPeersVersion() >= minVersion; + } + + // Common block data public boolean canUseCachedCommonBlockData() { diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index ba9678f2..f94db927 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -124,6 +124,8 @@ public class Settings { private int networkPoWComputePoolSize = 2; /** Maximum number of retry attempts if a peer fails to respond with the requested data */ private int maxRetries = 2; + /** Minimum peer version number required in order to sync with them */ + private String minPeerVersion = "1.5.0"; // Which blockchains this node is running private String blockchainConfig = null; // use default from resources @@ -412,6 +414,8 @@ public class Settings { public int getMaxRetries() { return this.maxRetries; } + public String getMinPeerVersion() { return this.minPeerVersion; } + public String getBlockchainConfig() { return this.blockchainConfig; } From f6ba5f5d51a5523bd21bfb6a235c7cddbebb8d85 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Apr 2021 17:49:04 +0100 Subject: [PATCH 09/15] Added /blocks/byheight/{height}/mintinginfo API, which returns info on the minter level, key distance, and block timings. --- .../qortal/api/model/BlockMintingInfo.java | 22 +++++++ .../qortal/api/resource/BlocksResource.java | 58 +++++++++++++++++++ src/main/java/org/qortal/block/Block.java | 2 +- 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/qortal/api/model/BlockMintingInfo.java diff --git a/src/main/java/org/qortal/api/model/BlockMintingInfo.java b/src/main/java/org/qortal/api/model/BlockMintingInfo.java new file mode 100644 index 00000000..e71c918b --- /dev/null +++ b/src/main/java/org/qortal/api/model/BlockMintingInfo.java @@ -0,0 +1,22 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.math.BigDecimal; +import java.math.BigInteger; + +@XmlAccessorType(XmlAccessType.FIELD) +public class BlockMintingInfo { + + public byte[] minterPublicKey; + public int minterLevel; + public BigDecimal maxDistance; + public BigInteger keyDistance; + public double keyDistanceRatio; + public long timestamp; + public long timeDelta; + + public BlockMintingInfo() { + } + +} diff --git a/src/main/java/org/qortal/api/resource/BlocksResource.java b/src/main/java/org/qortal/api/resource/BlocksResource.java index 30cc477e..1b160b91 100644 --- a/src/main/java/org/qortal/api/resource/BlocksResource.java +++ b/src/main/java/org/qortal/api/resource/BlocksResource.java @@ -8,6 +8,9 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; import java.util.ArrayList; import java.util.List; @@ -20,10 +23,13 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import org.qortal.account.Account; import org.qortal.api.ApiError; import org.qortal.api.ApiErrors; import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.model.BlockMintingInfo; import org.qortal.api.model.BlockSignerSummary; +import org.qortal.block.Block; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountData; import org.qortal.data.block.BlockData; @@ -328,6 +334,58 @@ public class BlocksResource { } } + @GET + @Path("/byheight/{height}/mintinginfo") + @Operation( + summary = "Fetch block minter info using block height", + description = "Returns the minter info for the block with given height", + responses = { + @ApiResponse( + description = "the block", + content = @Content( + schema = @Schema( + implementation = BlockData.class + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE + }) + public BlockMintingInfo getBlockMintingInfoByHeight(@PathParam("height") int height) { + try (final Repository repository = RepositoryManager.getRepository()) { + BlockData blockData = repository.getBlockRepository().fromHeight(height); + if (blockData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + + Block block = new Block(repository, blockData); + BlockData parentBlockData = repository.getBlockRepository().fromSignature(blockData.getReference()); + int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey()); + if (minterLevel == 0) + // This may be unavailable when requesting a trimmed block + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + BigInteger distance = block.calcKeyDistance(parentBlockData.getHeight(), parentBlockData.getSignature(), blockData.getMinterPublicKey(), minterLevel); + double ratio = new BigDecimal(distance).divide(new BigDecimal(block.MAX_DISTANCE), 40, RoundingMode.DOWN).doubleValue(); + long timestamp = block.calcTimestamp(parentBlockData, blockData.getMinterPublicKey(), minterLevel); + long timeDelta = timestamp - parentBlockData.getTimestamp(); + + BlockMintingInfo blockMintingInfo = new BlockMintingInfo(); + blockMintingInfo.minterPublicKey = blockData.getMinterPublicKey(); + blockMintingInfo.minterLevel = minterLevel; + blockMintingInfo.maxDistance = new BigDecimal(block.MAX_DISTANCE); + blockMintingInfo.keyDistance = distance; + blockMintingInfo.keyDistanceRatio = ratio; + blockMintingInfo.timestamp = timestamp; + blockMintingInfo.timeDelta = timeDelta; + + return blockMintingInfo; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + @GET @Path("/timestamp/{timestamp}") @Operation( diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index e2a15911..7f1f9da9 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -232,7 +232,7 @@ public class Block { // Other useful constants - private static final BigInteger MAX_DISTANCE; + public static final BigInteger MAX_DISTANCE; static { byte[] maxValue = new byte[Transformer.PUBLIC_KEY_LENGTH]; Arrays.fill(maxValue, (byte) 0xFF); From 019ab2b21d8e1e073f2a1a198210dfb067fc6efc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Apr 2021 17:57:28 +0100 Subject: [PATCH 10/15] Added tools/block-timings-sh which can be used to test out new block timings (specified in blockchain.json). The script will fetch a set of blocks and then backtest the specified blockTimings settings (target, deviation, and power) against those real life blocks. This allows configurations to be fine tuned to tighten up block times, and to adjust the timestamp variance between levels. Usage: block-timings.sh [target] [deviation] [power] startheight: a block height, preferably within the untrimmed range, to avoid data gaps count: the number of blocks to request and analyse after the start height. Default: 100 target: the target block time in milliseconds. Originates from blockchain.json. Default: 60000 deviation: the allowed block time deviation in milliseconds. Originates from blockchain.json. Default: 30000 power: used when transforming key distance to a time offset. Originates from blockchain.json. Default: 0.2 --- tools/block-timings.sh | 147 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100755 tools/block-timings.sh diff --git a/tools/block-timings.sh b/tools/block-timings.sh new file mode 100755 index 00000000..43f2f466 --- /dev/null +++ b/tools/block-timings.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash + +start_height=$1 +count=$2 + +if [ -z "${start_height}" ]; then + echo + echo "Error: missing start height." + echo + echo "Usage:" + echo "block-timings.sh [target] [deviation] [power]" + echo + echo "startheight: a block height, preferably within the untrimmed range, to avoid data gaps" + echo "count: the number of blocks to request and analyse after the start height. Default: 100" + echo "target: the target block time in milliseconds. Originates from blockchain.json. Default: 60000" + echo "deviation: the allowed block time deviation in milliseconds. Originates from blockchain.json. Default: 30000" + echo "power: used when transforming key distance to a time offset. Originates from blockchain.json. Default: 0.2" + echo + exit +fi + +target=$3 +deviation=$4 +power=$5 + +count=${count:=100} +target=${target:=60000} +deviation=${deviation:=30000} +power=${power:=0.2} + +finish_height=$((start_height + count - 1)) +height=$start_height + +echo "Settings:" +echo "Target time offset: ${target}" +echo "Deviation: ${deviation}" +echo "Power transform: ${power}" +echo + +function calculate_time_offset { + local key_distance_ratio=$1 + local transformed=$( echo "" | awk "END {print ${key_distance_ratio} ^ ${power}}") + local time_offset=$(echo "${deviation}*2*${transformed}" | bc) + time_offset=${time_offset%.*} + echo $time_offset +} + + +function fetch_and_process_blocks { + + echo "Fetching blocks from height ${start_height} to ${finish_height}..." + echo + + total_time_offset=0 + errors=0 + + while [ "${height}" -le "${finish_height}" ]; do + block_minting_info=$(curl -s "http://localhost:12391/blocks/byheight/${height}/mintinginfo") + error=$(echo "${block_minting_info}" | jq -r .error) + if [ "${error}" != "null" ]; then + echo "Error fetching minting info for block ${height}" + echo + errors=$((errors+1)) + height=$((height+1)) + continue; + fi + + # Parse minting info + minter_level=$(echo "${block_minting_info}" | jq -r .minterLevel) + key_distance_ratio=$(echo "${block_minting_info}" | jq -r .keyDistanceRatio) + time_delta=$(echo "${block_minting_info}" | jq -r .timeDelta) + + time_offset=$(calculate_time_offset "${key_distance_ratio}") + block_time=$((target-deviation+time_offset)) + + echo "=== BLOCK ${height} ===" + echo "Minter level: ${minter_level}" + echo "Key distance ratio: ${key_distance_ratio}" + echo "Time offset: ${time_offset}" + echo "Block time (real): ${time_delta}" + echo "Block time (calculated): ${block_time}" + + if [ "${time_delta}" -ne "${block_time}" ]; then + echo "WARNING: Block time mismatch. This is to be expected when using custom settings." + fi + echo + + total_time_offset=$((total_time_offset+block_time)) + + height=$((height+1)) + done + + adjusted_count=$((count-errors)) + if [ "${adjusted_count}" -eq 0 ]; then + echo "No blocks were retrieved." + echo + exit; + fi + + mean_time_offset=$((total_time_offset/adjusted_count)) + time_offset_diff=$((mean_time_offset-target)) + + echo "===================" + echo "===== SUMMARY =====" + echo "===================" + echo "Total blocks retrieved: ${adjusted_count}" + echo "Total blocks failed: ${errors}" + echo "Mean time offset: ${mean_time_offset}ms" + echo "Target time offset: ${target}ms" + echo "Difference from target: ${time_offset_diff}ms" + echo + +} + +function estimate_key_distance_ratio_for_level { + local level=$1 + local example_key_distance="0.5" + echo "(${example_key_distance}/${level})" +} + +function estimate_block_timestamps { + min_block_time=9999999 + max_block_time=0 + + echo "===== BLOCK TIME ESTIMATES =====" + + for level in {1..10}; do + example_key_distance_ratio=$(estimate_key_distance_ratio_for_level "${level}") + time_offset=$(calculate_time_offset "${example_key_distance_ratio}") + block_time=$((target-deviation+time_offset)) + + if [ "${block_time}" -gt "${max_block_time}" ]; then + max_block_time=${block_time} + fi + if [ "${block_time}" -lt "${min_block_time}" ]; then + min_block_time=${block_time} + fi + + echo "Level: ${level}, time offset: ${time_offset}, block time: ${block_time}" + done + block_time_range=$((max_block_time-min_block_time)) + echo "Range: ${block_time_range}" + echo +} + +fetch_and_process_blocks +estimate_block_timestamps From a06faa76857cae6f4af5f4170e0bee07b8997fdf Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Apr 2021 18:12:09 +0100 Subject: [PATCH 11/15] Updated usage info to reflect the fact that the "count" parameter is optional. Usage: block-timings.sh [count] [target] [deviation] [power] --- tools/block-timings.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/block-timings.sh b/tools/block-timings.sh index 43f2f466..514168dd 100755 --- a/tools/block-timings.sh +++ b/tools/block-timings.sh @@ -8,7 +8,7 @@ if [ -z "${start_height}" ]; then echo "Error: missing start height." echo echo "Usage:" - echo "block-timings.sh [target] [deviation] [power]" + echo "block-timings.sh [count] [target] [deviation] [power]" echo echo "startheight: a block height, preferably within the untrimmed range, to avoid data gaps" echo "count: the number of blocks to request and analyse after the start height. Default: 100" From 1c38afcd25bb9d123626dd0b95aa17be5c7f4b5e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Apr 2021 18:24:33 +0100 Subject: [PATCH 12/15] Slight reordering of vars. --- tools/block-timings.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/block-timings.sh b/tools/block-timings.sh index 514168dd..5ca4f08a 100755 --- a/tools/block-timings.sh +++ b/tools/block-timings.sh @@ -2,6 +2,9 @@ start_height=$1 count=$2 +target=$3 +deviation=$4 +power=$5 if [ -z "${start_height}" ]; then echo @@ -19,10 +22,6 @@ if [ -z "${start_height}" ]; then exit fi -target=$3 -deviation=$4 -power=$5 - count=${count:=100} target=${target:=60000} deviation=${deviation:=30000} From e300a957e4b6f79fc56a3ad1c913fb5914cd9f6e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 May 2021 19:25:05 +0100 Subject: [PATCH 13/15] Added online accounts count to /blocks/byheight/{height}/mintinginfo API and block-timings.sh script. --- src/main/java/org/qortal/api/model/BlockMintingInfo.java | 1 + src/main/java/org/qortal/api/resource/BlocksResource.java | 1 + tools/block-timings.sh | 2 ++ 3 files changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/api/model/BlockMintingInfo.java b/src/main/java/org/qortal/api/model/BlockMintingInfo.java index e71c918b..f84e179e 100644 --- a/src/main/java/org/qortal/api/model/BlockMintingInfo.java +++ b/src/main/java/org/qortal/api/model/BlockMintingInfo.java @@ -10,6 +10,7 @@ public class BlockMintingInfo { public byte[] minterPublicKey; public int minterLevel; + public int onlineAccountsCount; public BigDecimal maxDistance; public BigInteger keyDistance; public double keyDistanceRatio; diff --git a/src/main/java/org/qortal/api/resource/BlocksResource.java b/src/main/java/org/qortal/api/resource/BlocksResource.java index 1b160b91..b2f29305 100644 --- a/src/main/java/org/qortal/api/resource/BlocksResource.java +++ b/src/main/java/org/qortal/api/resource/BlocksResource.java @@ -374,6 +374,7 @@ public class BlocksResource { BlockMintingInfo blockMintingInfo = new BlockMintingInfo(); blockMintingInfo.minterPublicKey = blockData.getMinterPublicKey(); blockMintingInfo.minterLevel = minterLevel; + blockMintingInfo.onlineAccountsCount = blockData.getOnlineAccountsCount(); blockMintingInfo.maxDistance = new BigDecimal(block.MAX_DISTANCE); blockMintingInfo.keyDistance = distance; blockMintingInfo.keyDistanceRatio = ratio; diff --git a/tools/block-timings.sh b/tools/block-timings.sh index 5ca4f08a..5324209b 100755 --- a/tools/block-timings.sh +++ b/tools/block-timings.sh @@ -66,6 +66,7 @@ function fetch_and_process_blocks { # Parse minting info minter_level=$(echo "${block_minting_info}" | jq -r .minterLevel) + online_accounts_count=$(echo "${block_minting_info}" | jq -r .onlineAccountsCount) key_distance_ratio=$(echo "${block_minting_info}" | jq -r .keyDistanceRatio) time_delta=$(echo "${block_minting_info}" | jq -r .timeDelta) @@ -74,6 +75,7 @@ function fetch_and_process_blocks { echo "=== BLOCK ${height} ===" echo "Minter level: ${minter_level}" + echo "Online accounts: ${online_accounts_count}" echo "Key distance ratio: ${key_distance_ratio}" echo "Time offset: ${time_offset}" echo "Block time (real): ${time_delta}" From af9b536dd97a095b7c3c98cedbaf14af4ecc0f7a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 May 2021 23:00:51 +0100 Subject: [PATCH 14/15] Moved version check above getMinBlockchainPeers() check, so that nodes with old versions aren't counted. --- src/main/java/org/qortal/controller/Controller.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index c1386f53..9123a130 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -644,11 +644,15 @@ public class Controller extends Thread { // Disregard peers that don't have a recent block peers.removeIf(hasNoRecentBlock); + // Disregard peers that are on an old version + peers.removeIf(hasOldVersion); + checkRecoveryModeForPeers(peers); if (recoveryMode) { peers = Network.getInstance().getHandshakedPeers(); peers.removeIf(hasOnlyGenesisBlock); peers.removeIf(hasMisbehaved); + peers.removeIf(hasOldVersion); } // Check we have enough peers to potentially synchronize @@ -661,9 +665,6 @@ public class Controller extends Thread { // Disregard peers that are on the same block as last sync attempt and we didn't like their chain peers.removeIf(hasInferiorChainTip); - // Disregard peers that are on an old version - peers.removeIf(hasOldVersion); - final int peersBeforeComparison = peers.size(); // Request recent block summaries from the remaining peers, and locate our common block with each From 26c1793d857feb8cdc00da9492118ba40a0a056c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 10 May 2021 09:00:42 +0100 Subject: [PATCH 15/15] Added "allowConnectionsWithOlderPeerVersions" setting (default: true) This controls whether to allow connections with peers below minPeerVersion. If true, we won't sync with them but they can still sync with us, and will show in the peers list. This is the default, which allows older nodes to continue functioning, but prevents them from interfering with the sync behaviour of updated nodes. If false, sync will be blocked both ways, and they will not appear in the peers list at all. --- src/main/java/org/qortal/network/Handshake.java | 9 +++++++++ src/main/java/org/qortal/settings/Settings.java | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/src/main/java/org/qortal/network/Handshake.java b/src/main/java/org/qortal/network/Handshake.java index ed6d02db..8bee63a2 100644 --- a/src/main/java/org/qortal/network/Handshake.java +++ b/src/main/java/org/qortal/network/Handshake.java @@ -71,6 +71,15 @@ public enum Handshake { peer.setPeersConnectionTimestamp(peersConnectionTimestamp); peer.setPeersVersion(versionString, version); + if (Settings.getInstance().getAllowConnectionsWithOlderPeerVersions() == false) { + // Ensure the peer is running at least the minimum version allowed for connections + final String minPeerVersion = Settings.getInstance().getMinPeerVersion(); + if (peer.isAtLeastVersion(minPeerVersion) == false) { + LOGGER.info(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString)); + return null; + } + } + return CHALLENGE; } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index f94db927..164223c9 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -124,8 +124,13 @@ public class Settings { private int networkPoWComputePoolSize = 2; /** Maximum number of retry attempts if a peer fails to respond with the requested data */ private int maxRetries = 2; + /** Minimum peer version number required in order to sync with them */ private String minPeerVersion = "1.5.0"; + /** Whether to allow connections with peers below minPeerVersion + * If true, we won't sync with them but they can still sync with us, and will show in the peers list + * If false, sync will be blocked both ways, and they will not appear in the peers list */ + private boolean allowConnectionsWithOlderPeerVersions = true; // Which blockchains this node is running private String blockchainConfig = null; // use default from resources @@ -416,6 +421,8 @@ public class Settings { public String getMinPeerVersion() { return this.minPeerVersion; } + public boolean getAllowConnectionsWithOlderPeerVersions() { return this.allowConnectionsWithOlderPeerVersions; } + public String getBlockchainConfig() { return this.blockchainConfig; }