Qortal/qortal
Qortal/qortal
3
0
Fork 3
mirror of https://github.com/Qortal/qortal.git synced 2025-02-12 18:25:49 +00:00
Code Issues Projects Releases Wiki Activity

HTLC secret and status caching to reduce repeated calls to ElectrumX network

Browse Source 
Additional benefit is to speed up syncing if node has trade-bot entries,
as a batch of blocks processed within 30s will have the same HTLC responses
as neither Bitcoin nor Litecoin blocks are that fast. Once synced, the next
Qortal block (~60s) should pick up any new changes. Generally users will be
synced when using trade-bot anyway.

Also moved bitcoiny.getAddressTransactions(p2shAddress)
into BitcoinyHTLC.findHtlcSecret() so only called if secret,
or lack thereof, is not cached.

Added tests to cover caching.
This commit is contained in:
catbref 2020-09-22 12:13:22 +01:00
parent 4bc0edeeca
commit 7cd8ed6e23
11 changed files with 166 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java
View File

@ -159,8 +159,7 @@ public class CrossChainHtlcResource {
if (now >= medianBlockTime * 1000L) {
// See if we can extract secret
List<byte[]> rawTransactions = bitcoiny.getAddressTransactions(htlcStatus.bitcoinP2shAddress);
htlcStatus.secret = BitcoinyHTLC.findHtlcSecret(bitcoiny.getNetworkParameters(), htlcStatus.bitcoinP2shAddress, rawTransactions);
htlcStatus.secret = BitcoinyHTLC.findHtlcSecret(bitcoiny, htlcStatus.bitcoinP2shAddress);
}
return htlcStatus;

4
src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java
View File

@ -947,9 +947,7 @@ public class BitcoinACCTv1TradeBot implements AcctTradeBot {
return;
}
List<byte[]> p2shTransactions = bitcoin.getAddressTransactions(p2shAddressB);
byte[] secretB = BitcoinyHTLC.findHtlcSecret(bitcoin.getNetworkParameters(), p2shAddressB, p2shTransactions);
byte[] secretB = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddressB);
if (secretB == null)
// Secret not revealed at this time
return;

2
src/main/java/org/qortal/crosschain/Bitcoin.java
View File

@ -164,7 +164,7 @@ public class Bitcoin extends Bitcoiny {
if (instance == null) {
BitcoinNet bitcoinNet = Settings.getInstance().getBitcoinNet();
BitcoinyBlockchainProvider electrumX = new ElectrumX(bitcoinNet.getGenesisHash(), bitcoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS);
BitcoinyBlockchainProvider electrumX = new ElectrumX("Bitcoin-" + bitcoinNet.name(), bitcoinNet.getGenesisHash(), bitcoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS);
Context bitcoinjContext = new Context(bitcoinNet.getParams());
instance = new Bitcoin(bitcoinNet, electrumX, bitcoinjContext, CURRENCY_CODE);

3
src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java
View File

@ -7,6 +7,9 @@ public abstract class BitcoinyBlockchainProvider {
public static final boolean INCLUDE_UNCONFIRMED = true;
public static final boolean EXCLUDE_UNCONFIRMED = false;
/** Returns ID unique to bitcoiny network (e.g. "Litecoin-TEST3") */
public abstract String getNetId();
/** Returns current blockchain height. */
public abstract int getCurrentHeight() throws ForeignBlockchainException;

72
src/main/java/org/qortal/crosschain/BitcoinyHTLC.java
View File

@ -4,6 +4,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
@ -41,6 +42,31 @@ public class BitcoinyHTLC {
public static final long NO_LOCKTIME_NO_RBF_SEQUENCE = 0xFFFFFFFFL;
public static final long LOCKTIME_NO_RBF_SEQUENCE = NO_LOCKTIME_NO_RBF_SEQUENCE - 1;
// Assuming node's trade-bot has no more than 100 entries?
private static final int MAX_CACHE_ENTRIES = 100;
// Max time-to-live for cache entries (milliseconds)
private static final long CACHE_TIMEOUT = 30_000L;
@SuppressWarnings("serial")
private static final Map<String, byte[]> SECRET_CACHE = new LinkedHashMap<>(MAX_CACHE_ENTRIES + 1, 0.75F, true) {
// This method is called just after a new entry has been added
@Override
public boolean removeEldestEntry(Map.Entry<String, byte[]> eldest) {
return size() > MAX_CACHE_ENTRIES;
}
};
private static final byte[] NO_SECRET_CACHE_ENTRY = new byte[0];
@SuppressWarnings("serial")
private static final Map<String, Status> STATUS_CACHE = new LinkedHashMap<>(MAX_CACHE_ENTRIES + 1, 0.75F, true) {
// This method is called just after a new entry has been added
@Override
public boolean removeEldestEntry(Map.Entry<String, Status> eldest) {
return size() > MAX_CACHE_ENTRIES;
}
};
/*
* OP_TUCK (to copy public key to before signature)
* OP_CHECKSIGVERIFY (sig & pubkey must verify or script fails)
@ -207,8 +233,21 @@ public class BitcoinyHTLC {
return buildP2shTransaction(params, redeemAmount, redeemKey, fundingOutputs, redeemScriptBytes, null, redeemSigScriptBuilder, receivingAccountInfo);
}
/** Returns 'secret', if any, given list of raw transactions. */
public static byte[] findHtlcSecret(NetworkParameters params, String p2shAddress, List<byte[]> rawTransactions) {
/**
* Returns 'secret', if any, given HTLC's P2SH address.
* <p>
* @throws ForeignBlockchainException
*/
public static byte[] findHtlcSecret(Bitcoiny bitcoiny, String p2shAddress) throws ForeignBlockchainException {
NetworkParameters params = bitcoiny.getNetworkParameters();
String compoundKey = String.format("%s-%s-%d", params.getId(), p2shAddress, System.currentTimeMillis() / CACHE_TIMEOUT);
byte[] secret = SECRET_CACHE.getOrDefault(compoundKey, NO_SECRET_CACHE_ENTRY);
if (secret != NO_SECRET_CACHE_ENTRY)
return secret;
List<byte[]> rawTransactions = bitcoiny.getAddressTransactions(p2shAddress);
for (byte[] rawTransaction : rawTransactions) {
Transaction transaction = new Transaction(params, rawTransaction);
@ -237,14 +276,20 @@ public class BitcoinyHTLC {
// Input isn't spending our HTLC
continue;
byte[] secret = scriptChunks.get(0).data;
secret = scriptChunks.get(0).data;
if (secret.length != BitcoinyHTLC.SECRET_LENGTH)
continue;
// Cache secret for a while
SECRET_CACHE.put(compoundKey, secret);
return secret;
}
}
// Cache negative result
SECRET_CACHE.put(compoundKey, null);
return null;
}
@ -254,6 +299,12 @@ public class BitcoinyHTLC {
* @throws ForeignBlockchainException if error occurs
*/
public static Status determineHtlcStatus(BitcoinyBlockchainProvider blockchain, String p2shAddress, long minimumAmount) throws ForeignBlockchainException {
String compoundKey = String.format("%s-%s-%d", blockchain.getNetId(), p2shAddress, System.currentTimeMillis() / CACHE_TIMEOUT);
Status cachedStatus = STATUS_CACHE.getOrDefault(compoundKey, null);
if (cachedStatus != null)
return cachedStatus;
byte[] ourScriptPubKey = addressToScriptPubKey(p2shAddress);
List<TransactionHash> transactionHashes = blockchain.getAddressTransactions(ourScriptPubKey, BitcoinyBlockchainProvider.INCLUDE_UNCONFIRMED);
@ -293,9 +344,12 @@ public class BitcoinyHTLC {
if (scriptSigChunks.size() == 4)
// If we have 4 chunks, then secret is present, hence redeem
return transactionInfo.height == 0 ? Status.REDEEM_IN_PROGRESS : Status.REDEEMED;
cachedStatus = transactionInfo.height == 0 ? Status.REDEEM_IN_PROGRESS : Status.REDEEMED;
else
return transactionInfo.height == 0 ? Status.REFUND_IN_PROGRESS : Status.REFUNDED;
cachedStatus = transactionInfo.height == 0 ? Status.REFUND_IN_PROGRESS : Status.REFUNDED;
STATUS_CACHE.put(compoundKey, cachedStatus);
return cachedStatus;
}
String ourScriptPubKeyHex = HashCode.fromBytes(ourScriptPubKey).toString();
@ -319,11 +373,15 @@ public class BitcoinyHTLC {
// Not funding our specific P2SH
continue;
return transactionInfo.height == 0 ? Status.FUNDING_IN_PROGRESS : Status.FUNDED;
cachedStatus = transactionInfo.height == 0 ? Status.FUNDING_IN_PROGRESS : Status.FUNDED;
STATUS_CACHE.put(compoundKey, cachedStatus);
return cachedStatus;
}
}
return Status.UNFUNDED;
cachedStatus = Status.UNFUNDED;
STATUS_CACHE.put(compoundKey, cachedStatus);
return cachedStatus;
}
private static List<byte[]> extractScriptSigChunks(byte[] scriptSigBytes) {

17
src/main/java/org/qortal/crosschain/ElectrumX.java
View File

@ -85,6 +85,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
private Set<Server> servers = new HashSet<>();
private List<Server> remainingServers = new ArrayList<>();
private final String netId;
private final String expectedGenesisHash;
private final Map<Server.ConnectionType, Integer> defaultPorts = new EnumMap<>(Server.ConnectionType.class);
@ -95,7 +96,8 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
// Constructors
public ElectrumX(String genesisHash, Collection<Server> initialServerList, Map<Server.ConnectionType, Integer> defaultPorts) {
public ElectrumX(String netId, String genesisHash, Collection<Server> initialServerList, Map<Server.ConnectionType, Integer> defaultPorts) {
this.netId = netId;
this.expectedGenesisHash = genesisHash;
this.servers.addAll(initialServerList);
this.defaultPorts.putAll(defaultPorts);
@ -103,11 +105,17 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
// Methods for use by other classes
@Override
public String getNetId() {
return this.netId;
}
/**
* Returns current blockchain height.
* <p>
* @throws ForeignBlockchainException if error occurs
*/
@Override
public int getCurrentHeight() throws ForeignBlockchainException {
Object blockObj = this.rpc("blockchain.headers.subscribe");
if (!(blockObj instanceof JSONObject))
@ -128,6 +136,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
* <p>
* @throws ForeignBlockchainException if error occurs
*/
@Override
public List<byte[]> getRawBlockHeaders(int startHeight, int count) throws ForeignBlockchainException {
Object blockObj = this.rpc("blockchain.block.headers", startHeight, count);
if (!(blockObj instanceof JSONObject))
@ -161,6 +170,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
* @return confirmed balance, or zero if script unknown
* @throws ForeignBlockchainException if there was an error
*/
@Override
public long getConfirmedBalance(byte[] script) throws ForeignBlockchainException {
byte[] scriptHash = Crypto.digest(script);
Bytes.reverse(scriptHash);
@ -185,6 +195,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
* @return list of unspent outputs, or empty list if script unknown
* @throws ForeignBlockchainException if there was an error.
*/
@Override
public List<UnspentOutput> getUnspentOutputs(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException {
byte[] scriptHash = Crypto.digest(script);
Bytes.reverse(scriptHash);
@ -218,6 +229,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
* @throws ForeignBlockchainException.NotFoundException if transaction not found
* @throws ForeignBlockchainException if error occurs
*/
@Override
public byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException {
Object rawTransactionHex;
try {
@ -242,6 +254,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
* @throws ForeignBlockchainException.NotFoundException if transaction not found
* @throws ForeignBlockchainException if error occurs
*/
@Override
public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException {
Object transactionObj;
try {
@ -317,6 +330,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
* @return list of related transactions, or empty list if script unknown
* @throws ForeignBlockchainException if error occurs
*/
@Override
public List<TransactionHash> getAddressTransactions(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException {
byte[] scriptHash = Crypto.digest(script);
Bytes.reverse(scriptHash);
@ -348,6 +362,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
* <p>
* @throws ForeignBlockchainException if error occurs
*/
@Override
public void broadcastTransaction(byte[] transactionBytes) throws ForeignBlockchainException {
Object rawBroadcastResult = this.rpc("blockchain.transaction.broadcast", HashCode.fromBytes(transactionBytes).toString());

2
src/main/java/org/qortal/crosschain/Litecoin.java
View File

@ -138,7 +138,7 @@ public class Litecoin extends Bitcoiny {
if (instance == null) {
LitecoinNet litecoinNet = Settings.getInstance().getLitecoinNet();
BitcoinyBlockchainProvider electrumX = new ElectrumX(litecoinNet.getGenesisHash(), litecoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS);
BitcoinyBlockchainProvider electrumX = new ElectrumX("Litecoin-" + litecoinNet.name(), litecoinNet.getGenesisHash(), litecoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS);
Context bitcoinjContext = new Context(litecoinNet.getParams());
instance = new Litecoin(litecoinNet, electrumX, bitcoinjContext, CURRENCY_CODE);

5
src/test/java/org/qortal/test/crosschain/BitcoinTests.java
View File

@ -3,7 +3,6 @@ package org.qortal.test.crosschain;
import static org.junit.Assert.*;
import java.util.Arrays;
import java.util.List;
import org.bitcoinj.core.Transaction;
import org.bitcoinj.store.BlockStoreException;
@ -58,10 +57,8 @@ public class BitcoinTests extends Common {
// This actually exists on TEST3 but can take a while to fetch
String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
List<byte[]> rawTransactions = bitcoin.getAddressTransactions(p2shAddress);
byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes();
byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin.getNetworkParameters(), p2shAddress, rawTransactions);
byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress);
assertNotNull(secret);
assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret));

2
src/test/java/org/qortal/test/crosschain/ElectrumXTests.java
View File

@ -41,7 +41,7 @@ public class ElectrumXTests {
}
private ElectrumX getInstance() {
return new ElectrumX(BitcoinNet.TEST3.getGenesisHash(), BitcoinNet.TEST3.getServers(), DEFAULT_ELECTRUMX_PORTS);
return new ElectrumX("Bitcoin-" + BitcoinNet.TEST3.name(), BitcoinNet.TEST3.getGenesisHash(), BitcoinNet.TEST3.getServers(), DEFAULT_ELECTRUMX_PORTS);
}
@Test

82
src/test/java/org/qortal/test/crosschain/HtlcTests.java
View File

@ -2,18 +2,18 @@ package org.qortal.test.crosschain;
import static org.junit.Assert.*;
import java.util.Arrays;
import java.util.List;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.qortal.crosschain.Bitcoin;
import org.qortal.crosschain.ForeignBlockchainException;
import org.qortal.crypto.Crypto;
import org.qortal.crosschain.BitcoinyHTLC;
import org.qortal.repository.DataException;
import org.qortal.test.common.Common;
import com.google.common.primitives.Longs;
public class HtlcTests extends Common {
private Bitcoin bitcoin;
@ -35,13 +35,48 @@ public class HtlcTests extends Common {
// This actually exists on TEST3 but can take a while to fetch
String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
List<byte[]> rawTransactions = bitcoin.getAddressTransactions(p2shAddress);
byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes();
byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin.getNetworkParameters(), p2shAddress, rawTransactions);
byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress);
assertNotNull(secret);
assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret));
assertArrayEquals("secret incorrect", expectedSecret, secret);
}
@Test
public void testHtlcSecretCaching() throws ForeignBlockchainException {
String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes();
do {
// We need to perform fresh setup for 1st test
Bitcoin.resetForTesting();
bitcoin = Bitcoin.getInstance();
long now = System.currentTimeMillis();
long timestampBoundary = now / 30_000L;
byte[] secret1 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress);
long executionPeriod1 = System.currentTimeMillis() - now;
assertNotNull(secret1);
assertArrayEquals("secret1 incorrect", expectedSecret, secret1);
assertTrue("1st execution period should not be instant!", executionPeriod1 > 10);
byte[] secret2 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress);
long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1;
assertNotNull(secret2);
assertArrayEquals("secret2 incorrect", expectedSecret, secret2);
// Test is only valid if we've called within same timestampBoundary
if (System.currentTimeMillis() / 30_000L != timestampBoundary)
continue;
assertArrayEquals(secret1, secret2);
assertTrue("2st execution period should be effectively instant!", executionPeriod2 < 10);
} while (false);
}
@Test
@ -55,4 +90,37 @@ public class HtlcTests extends Common {
System.out.println(String.format("HTLC %s status: %s", p2shAddress, htlcStatus.name()));
}
@Test
public void testHtlcStatusCaching() throws ForeignBlockchainException {
do {
// We need to perform fresh setup for 1st test
Bitcoin.resetForTesting();
bitcoin = Bitcoin.getInstance();
long now = System.currentTimeMillis();
long timestampBoundary = now / 30_000L;
// Won't ever exist
String p2shAddress = bitcoin.deriveP2shAddress(Crypto.hash160(Longs.toByteArray(now)));
BitcoinyHTLC.Status htlcStatus1 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L);
long executionPeriod1 = System.currentTimeMillis() - now;
assertNotNull(htlcStatus1);
assertTrue("1st execution period should not be instant!", executionPeriod1 > 10);
BitcoinyHTLC.Status htlcStatus2 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L);
long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1;
assertNotNull(htlcStatus2);
assertEquals(htlcStatus1, htlcStatus2);
// Test is only valid if we've called within same timestampBoundary
if (System.currentTimeMillis() / 30_000L != timestampBoundary)
continue;
assertTrue("2st execution period should be effectively instant!", executionPeriod2 < 10);
} while (false);
}
}

5
src/test/java/org/qortal/test/crosschain/LitecoinTests.java
View File

@ -3,7 +3,6 @@ package org.qortal.test.crosschain;
import static org.junit.Assert.*;
import java.util.Arrays;
import java.util.List;
import org.bitcoinj.core.Transaction;
import org.bitcoinj.store.BlockStoreException;
@ -55,10 +54,8 @@ public class LitecoinTests extends Common {
// This actually exists on TEST3 but can take a while to fetch
String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
List<byte[]> rawTransactions = litecoin.getAddressTransactions(p2shAddress);
byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes();
byte[] secret = BitcoinyHTLC.findHtlcSecret(litecoin.getNetworkParameters(), p2shAddress, rawTransactions);
byte[] secret = BitcoinyHTLC.findHtlcSecret(litecoin, p2shAddress);
assertNotNull("secret not found", secret);
assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret));