HTLC secret and status caching to reduce repeated calls to ElectrumX network

Additional benefit is to speed up syncing if node has trade-bot entries,
as a batch of blocks processed within 30s will have the same HTLC responses
as neither Bitcoin nor Litecoin blocks are that fast. Once synced, the next
Qortal block (~60s) should pick up any new changes. Generally users will be
synced when using trade-bot anyway.

Also moved bitcoiny.getAddressTransactions(p2shAddress)
into BitcoinyHTLC.findHtlcSecret() so only called if secret,
or lack thereof, is not cached.

Added tests to cover caching.

src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java

@@ -159,8 +159,7 @@ public class CrossChainHtlcResource {
 
 			if (now >= medianBlockTime * 1000L) {
 				// See if we can extract secret
-				List<byte[]> rawTransactions = bitcoiny.getAddressTransactions(htlcStatus.bitcoinP2shAddress);
+				htlcStatus.secret = BitcoinyHTLC.findHtlcSecret(bitcoiny, htlcStatus.bitcoinP2shAddress);
-				htlcStatus.secret = BitcoinyHTLC.findHtlcSecret(bitcoiny.getNetworkParameters(), htlcStatus.bitcoinP2shAddress, rawTransactions);
 			}
 
 			return htlcStatus;

src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java

@@ -947,9 +947,7 @@ public class BitcoinACCTv1TradeBot implements AcctTradeBot {
 				return;
 		}
 
-		List<byte[]> p2shTransactions = bitcoin.getAddressTransactions(p2shAddressB);
+		byte[] secretB = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddressB);
-
-		byte[] secretB = BitcoinyHTLC.findHtlcSecret(bitcoin.getNetworkParameters(), p2shAddressB, p2shTransactions);
 		if (secretB == null)
 			// Secret not revealed at this time
 			return;

src/main/java/org/qortal/crosschain/Bitcoin.java

@@ -164,7 +164,7 @@ public class Bitcoin extends Bitcoiny {
 		if (instance == null) {
 			BitcoinNet bitcoinNet = Settings.getInstance().getBitcoinNet();
 
-			BitcoinyBlockchainProvider electrumX = new ElectrumX(bitcoinNet.getGenesisHash(), bitcoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS);
+			BitcoinyBlockchainProvider electrumX = new ElectrumX("Bitcoin-" + bitcoinNet.name(), bitcoinNet.getGenesisHash(), bitcoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS);
 			Context bitcoinjContext = new Context(bitcoinNet.getParams());
 
 			instance = new Bitcoin(bitcoinNet, electrumX, bitcoinjContext, CURRENCY_CODE);

src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java

@@ -7,6 +7,9 @@ public abstract class BitcoinyBlockchainProvider {
 	public static final boolean INCLUDE_UNCONFIRMED = true;
 	public static final boolean EXCLUDE_UNCONFIRMED = false;
 
+	/** Returns ID unique to bitcoiny network (e.g. "Litecoin-TEST3") */
+	public abstract String getNetId();
+
 	/** Returns current blockchain height. */
 	public abstract int getCurrentHeight() throws ForeignBlockchainException;
 

src/main/java/org/qortal/crosschain/BitcoinyHTLC.java

@@ -4,6 +4,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
@@ -41,6 +42,31 @@ public class BitcoinyHTLC {
 	public static final long NO_LOCKTIME_NO_RBF_SEQUENCE = 0xFFFFFFFFL;
 	public static final long LOCKTIME_NO_RBF_SEQUENCE = NO_LOCKTIME_NO_RBF_SEQUENCE - 1;
 
+	// Assuming node's trade-bot has no more than 100 entries?
+	private static final int MAX_CACHE_ENTRIES = 100;
+
+	// Max time-to-live for cache entries (milliseconds)
+	private static final long CACHE_TIMEOUT = 30_000L;
+
+	@SuppressWarnings("serial")
+	private static final Map<String, byte[]> SECRET_CACHE = new LinkedHashMap<>(MAX_CACHE_ENTRIES + 1, 0.75F, true) {
+		// This method is called just after a new entry has been added
+		@Override
+		public boolean removeEldestEntry(Map.Entry<String, byte[]> eldest) {
+			return size() > MAX_CACHE_ENTRIES;
+		}
+	};
+	private static final byte[] NO_SECRET_CACHE_ENTRY = new byte[0];
+
+	@SuppressWarnings("serial")
+	private static final Map<String, Status> STATUS_CACHE = new LinkedHashMap<>(MAX_CACHE_ENTRIES + 1, 0.75F, true) {
+		// This method is called just after a new entry has been added
+		@Override
+		public boolean removeEldestEntry(Map.Entry<String, Status> eldest) {
+			return size() > MAX_CACHE_ENTRIES;
+		}
+	};
+
 	/*
 	 * OP_TUCK (to copy public key to before signature)
 	 * OP_CHECKSIGVERIFY (sig & pubkey must verify or script fails)
@@ -207,8 +233,21 @@ public class BitcoinyHTLC {
 		return buildP2shTransaction(params, redeemAmount, redeemKey, fundingOutputs, redeemScriptBytes, null, redeemSigScriptBuilder, receivingAccountInfo);
 	}
 
-	/** Returns 'secret', if any, given list of raw transactions. */
+	/**
-	public static byte[] findHtlcSecret(NetworkParameters params, String p2shAddress, List<byte[]> rawTransactions) {
+	 * Returns 'secret', if any, given HTLC's P2SH address.
+	 * <p>
+	 * @throws ForeignBlockchainException
+	 */
+	public static byte[] findHtlcSecret(Bitcoiny bitcoiny, String p2shAddress) throws ForeignBlockchainException {
+		NetworkParameters params = bitcoiny.getNetworkParameters();
+		String compoundKey = String.format("%s-%s-%d", params.getId(), p2shAddress, System.currentTimeMillis() / CACHE_TIMEOUT);
+
+		byte[] secret = SECRET_CACHE.getOrDefault(compoundKey, NO_SECRET_CACHE_ENTRY);
+		if (secret != NO_SECRET_CACHE_ENTRY)
+			return secret;
+
+		List<byte[]> rawTransactions = bitcoiny.getAddressTransactions(p2shAddress);
+
 		for (byte[] rawTransaction : rawTransactions) {
 			Transaction transaction = new Transaction(params, rawTransaction);
 
@@ -237,14 +276,20 @@ public class BitcoinyHTLC {
 					// Input isn't spending our HTLC
 					continue;
 
-				byte[] secret = scriptChunks.get(0).data;
+				secret = scriptChunks.get(0).data;
 				if (secret.length != BitcoinyHTLC.SECRET_LENGTH)
 					continue;
 
+				// Cache secret for a while
+				SECRET_CACHE.put(compoundKey, secret);
+
 				return secret;
 			}
 		}
 
+		// Cache negative result
+		SECRET_CACHE.put(compoundKey, null);
+
 		return null;
 	}
 
@@ -254,6 +299,12 @@ public class BitcoinyHTLC {
 	 * @throws ForeignBlockchainException if error occurs
 	 */
 	public static Status determineHtlcStatus(BitcoinyBlockchainProvider blockchain, String p2shAddress, long minimumAmount) throws ForeignBlockchainException {
+		String compoundKey = String.format("%s-%s-%d", blockchain.getNetId(), p2shAddress, System.currentTimeMillis() / CACHE_TIMEOUT);
+
+		Status cachedStatus = STATUS_CACHE.getOrDefault(compoundKey, null);
+		if (cachedStatus != null)
+			return cachedStatus;
+
 		byte[] ourScriptPubKey = addressToScriptPubKey(p2shAddress);
 		List<TransactionHash> transactionHashes = blockchain.getAddressTransactions(ourScriptPubKey, BitcoinyBlockchainProvider.INCLUDE_UNCONFIRMED);
 
@@ -293,9 +344,12 @@ public class BitcoinyHTLC {
 
 			if (scriptSigChunks.size() == 4)
 				// If we have 4 chunks, then secret is present, hence redeem
-				return transactionInfo.height == 0 ? Status.REDEEM_IN_PROGRESS : Status.REDEEMED;
+				cachedStatus = transactionInfo.height == 0 ? Status.REDEEM_IN_PROGRESS : Status.REDEEMED;
 			else
-				return transactionInfo.height == 0 ? Status.REFUND_IN_PROGRESS : Status.REFUNDED;
+				cachedStatus = transactionInfo.height == 0 ? Status.REFUND_IN_PROGRESS : Status.REFUNDED;
+
+			STATUS_CACHE.put(compoundKey, cachedStatus);
+			return cachedStatus;
 		}
 
 		String ourScriptPubKeyHex = HashCode.fromBytes(ourScriptPubKey).toString();
@@ -319,11 +373,15 @@ public class BitcoinyHTLC {
 					// Not funding our specific P2SH
 					continue;
 
-				return transactionInfo.height == 0 ? Status.FUNDING_IN_PROGRESS : Status.FUNDED;
+				cachedStatus = transactionInfo.height == 0 ? Status.FUNDING_IN_PROGRESS : Status.FUNDED;
+				STATUS_CACHE.put(compoundKey, cachedStatus);
+				return cachedStatus;
 			}
 		}
 
-		return Status.UNFUNDED;
+		cachedStatus = Status.UNFUNDED;
+		STATUS_CACHE.put(compoundKey, cachedStatus);
+		return cachedStatus;
 	}
 
 	private static List<byte[]> extractScriptSigChunks(byte[] scriptSigBytes) {

src/main/java/org/qortal/crosschain/ElectrumX.java

@@ -85,6 +85,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	private Set<Server> servers = new HashSet<>();
 	private List<Server> remainingServers = new ArrayList<>();
 
+	private final String netId;
 	private final String expectedGenesisHash;
 	private final Map<Server.ConnectionType, Integer> defaultPorts = new EnumMap<>(Server.ConnectionType.class);
 
@@ -95,7 +96,8 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 
 	// Constructors
 
-	public ElectrumX(String genesisHash, Collection<Server> initialServerList, Map<Server.ConnectionType, Integer> defaultPorts) {
+	public ElectrumX(String netId, String genesisHash, Collection<Server> initialServerList, Map<Server.ConnectionType, Integer> defaultPorts) {
+		this.netId = netId;
 		this.expectedGenesisHash = genesisHash;
 		this.servers.addAll(initialServerList);
 		this.defaultPorts.putAll(defaultPorts);
@@ -103,11 +105,17 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 
 	// Methods for use by other classes
 
+	@Override
+	public String getNetId() {
+		return this.netId;
+	}
+
 	/**
 	 * Returns current blockchain height.
 	 * <p>
 	 * @throws ForeignBlockchainException if error occurs
 	 */
+	@Override
 	public int getCurrentHeight() throws ForeignBlockchainException {
 		Object blockObj = this.rpc("blockchain.headers.subscribe");
 		if (!(blockObj instanceof JSONObject))
@@ -128,6 +136,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	 * <p>
 	 * @throws ForeignBlockchainException if error occurs
 	 */
+	@Override
 	public List<byte[]> getRawBlockHeaders(int startHeight, int count) throws ForeignBlockchainException {
 		Object blockObj = this.rpc("blockchain.block.headers", startHeight, count);
 		if (!(blockObj instanceof JSONObject))
@@ -161,6 +170,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	 * @return confirmed balance, or zero if script unknown
 	 * @throws ForeignBlockchainException if there was an error
 	 */
+	@Override
 	public long getConfirmedBalance(byte[] script) throws ForeignBlockchainException {
 		byte[] scriptHash = Crypto.digest(script);
 		Bytes.reverse(scriptHash);
@@ -185,6 +195,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	 * @return list of unspent outputs, or empty list if script unknown
 	 * @throws ForeignBlockchainException if there was an error.
 	 */
+	@Override
 	public List<UnspentOutput> getUnspentOutputs(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException {
 		byte[] scriptHash = Crypto.digest(script);
 		Bytes.reverse(scriptHash);
@@ -218,6 +229,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	 * @throws ForeignBlockchainException.NotFoundException if transaction not found
 	 * @throws ForeignBlockchainException if error occurs
 	 */
+	@Override
 	public byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException {
 		Object rawTransactionHex;
 		try {
@@ -242,6 +254,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	 * @throws ForeignBlockchainException.NotFoundException if transaction not found
 	 * @throws ForeignBlockchainException if error occurs
 	 */
+	@Override
 	public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException {
 		Object transactionObj;
 		try {
@@ -317,6 +330,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	 * @return list of related transactions, or empty list if script unknown
 	 * @throws ForeignBlockchainException if error occurs
 	 */
+	@Override
 	public List<TransactionHash> getAddressTransactions(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException {
 		byte[] scriptHash = Crypto.digest(script);
 		Bytes.reverse(scriptHash);
@@ -348,6 +362,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	 * <p>
 	 * @throws ForeignBlockchainException if error occurs
 	 */
+	@Override
 	public void broadcastTransaction(byte[] transactionBytes) throws ForeignBlockchainException {
 		Object rawBroadcastResult = this.rpc("blockchain.transaction.broadcast", HashCode.fromBytes(transactionBytes).toString());
 

src/main/java/org/qortal/crosschain/Litecoin.java

@@ -138,7 +138,7 @@ public class Litecoin extends Bitcoiny {
 		if (instance == null) {
 			LitecoinNet litecoinNet = Settings.getInstance().getLitecoinNet();
 
-			BitcoinyBlockchainProvider electrumX = new ElectrumX(litecoinNet.getGenesisHash(), litecoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS);
+			BitcoinyBlockchainProvider electrumX = new ElectrumX("Litecoin-" + litecoinNet.name(), litecoinNet.getGenesisHash(), litecoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS);
 			Context bitcoinjContext = new Context(litecoinNet.getParams());
 
 			instance = new Litecoin(litecoinNet, electrumX, bitcoinjContext, CURRENCY_CODE);

src/test/java/org/qortal/test/crosschain/BitcoinTests.java

@@ -3,7 +3,6 @@ package org.qortal.test.crosschain;
 import static org.junit.Assert.*;
 
 import java.util.Arrays;
-import java.util.List;
 
 import org.bitcoinj.core.Transaction;
 import org.bitcoinj.store.BlockStoreException;
@@ -58,10 +57,8 @@ public class BitcoinTests extends Common {
 		// This actually exists on TEST3 but can take a while to fetch
 		String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
 
-		List<byte[]> rawTransactions = bitcoin.getAddressTransactions(p2shAddress);
-
 		byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes();
-		byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin.getNetworkParameters(), p2shAddress, rawTransactions);
+		byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress);
 
 		assertNotNull(secret);
 		assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret));

src/test/java/org/qortal/test/crosschain/ElectrumXTests.java

@@ -41,7 +41,7 @@ public class ElectrumXTests {
 	}
 
 	private ElectrumX getInstance() {
-		return new ElectrumX(BitcoinNet.TEST3.getGenesisHash(), BitcoinNet.TEST3.getServers(), DEFAULT_ELECTRUMX_PORTS);
+		return new ElectrumX("Bitcoin-" + BitcoinNet.TEST3.name(), BitcoinNet.TEST3.getGenesisHash(), BitcoinNet.TEST3.getServers(), DEFAULT_ELECTRUMX_PORTS);
 	}
 
 	@Test

src/test/java/org/qortal/test/crosschain/HtlcTests.java

@@ -2,18 +2,18 @@ package org.qortal.test.crosschain;
 
 import static org.junit.Assert.*;
 
-import java.util.Arrays;
-import java.util.List;
-
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.qortal.crosschain.Bitcoin;
 import org.qortal.crosschain.ForeignBlockchainException;
+import org.qortal.crypto.Crypto;
 import org.qortal.crosschain.BitcoinyHTLC;
 import org.qortal.repository.DataException;
 import org.qortal.test.common.Common;
 
+import com.google.common.primitives.Longs;
+
 public class HtlcTests extends Common {
 
 	private Bitcoin bitcoin;
@@ -35,13 +35,48 @@ public class HtlcTests extends Common {
 		// This actually exists on TEST3 but can take a while to fetch
 		String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
 
-		List<byte[]> rawTransactions = bitcoin.getAddressTransactions(p2shAddress);
-
 		byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes();
-		byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin.getNetworkParameters(), p2shAddress, rawTransactions);
+		byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress);
 
 		assertNotNull(secret);
-		assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret));
+		assertArrayEquals("secret incorrect", expectedSecret, secret);
+	}
+
+	@Test
+	public void testHtlcSecretCaching() throws ForeignBlockchainException {
+		String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
+		byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes();
+
+		do {
+			// We need to perform fresh setup for 1st test
+			Bitcoin.resetForTesting();
+			bitcoin = Bitcoin.getInstance();
+
+			long now = System.currentTimeMillis();
+			long timestampBoundary = now / 30_000L;
+
+			byte[] secret1 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress);
+			long executionPeriod1 = System.currentTimeMillis() - now;
+
+			assertNotNull(secret1);
+			assertArrayEquals("secret1 incorrect", expectedSecret, secret1);
+
+			assertTrue("1st execution period should not be instant!", executionPeriod1 > 10);
+
+			byte[] secret2 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress);
+			long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1;
+
+			assertNotNull(secret2);
+			assertArrayEquals("secret2 incorrect", expectedSecret, secret2);
+
+			// Test is only valid if we've called within same timestampBoundary
+			if (System.currentTimeMillis() / 30_000L != timestampBoundary)
+				continue;
+
+			assertArrayEquals(secret1, secret2);
+
+			assertTrue("2st execution period should be effectively instant!", executionPeriod2 < 10);
+		} while (false);
 	}
 
 	@Test
@@ -55,4 +90,37 @@ public class HtlcTests extends Common {
 		System.out.println(String.format("HTLC %s status: %s", p2shAddress, htlcStatus.name()));
 	}
 
+	@Test
+	public void testHtlcStatusCaching() throws ForeignBlockchainException {
+		do {
+			// We need to perform fresh setup for 1st test
+			Bitcoin.resetForTesting();
+			bitcoin = Bitcoin.getInstance();
+
+			long now = System.currentTimeMillis();
+			long timestampBoundary = now / 30_000L;
+
+			// Won't ever exist
+			String p2shAddress = bitcoin.deriveP2shAddress(Crypto.hash160(Longs.toByteArray(now)));
+
+			BitcoinyHTLC.Status htlcStatus1 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L);
+			long executionPeriod1 = System.currentTimeMillis() - now;
+
+			assertNotNull(htlcStatus1);
+			assertTrue("1st execution period should not be instant!", executionPeriod1 > 10);
+
+			BitcoinyHTLC.Status htlcStatus2 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L);
+			long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1;
+
+			assertNotNull(htlcStatus2);
+			assertEquals(htlcStatus1, htlcStatus2);
+
+			// Test is only valid if we've called within same timestampBoundary
+			if (System.currentTimeMillis() / 30_000L != timestampBoundary)
+				continue;
+
+			assertTrue("2st execution period should be effectively instant!", executionPeriod2 < 10);
+		} while (false);
+	}
+
 }

src/test/java/org/qortal/test/crosschain/LitecoinTests.java

@@ -3,7 +3,6 @@ package org.qortal.test.crosschain;
 import static org.junit.Assert.*;
 
 import java.util.Arrays;
-import java.util.List;
 
 import org.bitcoinj.core.Transaction;
 import org.bitcoinj.store.BlockStoreException;
@@ -55,10 +54,8 @@ public class LitecoinTests extends Common {
 		// This actually exists on TEST3 but can take a while to fetch
 		String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
 
-		List<byte[]> rawTransactions = litecoin.getAddressTransactions(p2shAddress);
-
 		byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes();
-		byte[] secret = BitcoinyHTLC.findHtlcSecret(litecoin.getNetworkParameters(), p2shAddress, rawTransactions);
+		byte[] secret = BitcoinyHTLC.findHtlcSecret(litecoin, p2shAddress);
 
 		assertNotNull("secret not found", secret);
 		assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret));