Account lastReference cache, now with Block support.

As this changes how lastReferences are checked and updated,
this is not suitable for rolling into current chain without a
"feature trigger", or chain restart!

Added unit tests.

src/main/java/org/qortal/account/AccountRefCache.java

@@ -3,20 +3,66 @@ package org.qortal.account;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.function.BiFunction;
+import java.util.function.BinaryOperator;
 
 import org.qortal.data.account.AccountData;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.utils.Pair;
 
+/**
+ * Account lastReference caching
+ * <p>
+ * When checking an account's lastReference, the value returned should be the
+ * most recent value set after processing the most recent block.
+ * <p>
+ * However, when processing a batch of transactions, e.g. during block processing or validation,
+ * each transaction needs to check, and maybe update, multiple accounts' lastReference values.
+ * <p>
+ * Because the intermediate updates would affect future checks, we set up a cache of that
+ * maintains a consistent value for fetching lastReference, but also tracks the latest new
+ * value, without the overhead of repository calls.
+ * <p>
+ * Thus, when batch transaction processing is finished, only the latest new lastReference values
+ * can be committed to the repository, via {@link AccountRefCache#commit()}.
+ * <p>
+ * Getting and setting lastReferences values are done the usual way via
+ * {@link Account#getLastReference()} and {@link Account#setLastReference(byte[])} which call
+ * package-visibility methods in <tt>AccountRefCache</tt>.
+ * <p>
+ * If {@link Account#getLastReference()} or {@link Account#setLastReference(byte[])} are called
+ * outside of caching then lastReference values are fetched/set directly from/to the repository.
+ * <p>
+ * <tt>AccountRefCache</tt> implements <tt>AutoCloseable</tt> for (typical) use in a try-with-resources block.
+ *
+ * @see Account#getLastReference()
+ * @see Account#setLastReference(byte[])
+ * @see org.qortal.block.Block#process()
+ */
 public class AccountRefCache implements AutoCloseable {
 
-	private static final Map<Repository, RefCache> CACHE = new HashMap<Repository, RefCache>();
+	private static final Map<Repository, RefCache> CACHE = new HashMap<>();
 
 	private static class RefCache {
-		private final Map<String, byte[]> getLastReferenceValues = new HashMap<String, byte[]>();
+		private final Map<String, byte[]> getLastReferenceValues = new HashMap<>();
-		private final Map<String, Pair<byte[], byte[]>> setLastReferenceValues = new HashMap<String, Pair<byte[], byte[]>>();
+		private final Map<String, Pair<byte[], byte[]>> setLastReferenceValues = new HashMap<>();
+
+		/**
+		 * Function for merging publicKey from new data with old publicKey from map.
+		 * <p>
+		 * Last reference is <tt>A</tt> element in pair.<br>
+		 * Public key is <tt>B</tt> element in pair.
+		 */
+		private static final BinaryOperator<Pair<byte[], byte[]>> mergePublicKey = (oldPair,  newPair) -> {
+			// If passed new pair contains non-null publicKey, then we use that one in preference.
+			if (newPair.getB() == null)
+				// Otherwise, inherit publicKey from old map value.
+				newPair.setB(oldPair.getB());
+
+			// We always use new lastReference from new pair.
+			return newPair;
+		};
+
 
 		public byte[] getLastReference(Repository repository, String address) throws DataException {
 			synchronized (this.getLastReferenceValues) {
@@ -36,13 +82,11 @@ public class AccountRefCache implements AutoCloseable {
 		}
 
 		public void setLastReference(AccountData accountData) {
-			BiFunction<String, Pair<byte[], byte[]>, Pair<byte[], byte[]>> mergePublicKey = (key, oldPair) -> {
+			// We're only interested in lastReference and publicKey
-				byte[] mergedPublicKey = accountData.getPublicKey() != null ? accountData.getPublicKey() : oldPair.getB();
+			Pair<byte[], byte[]> newPair = new Pair<>(accountData.getReference(), accountData.getPublicKey());
-				return new Pair<>(accountData.getReference(), mergedPublicKey);
-			};
 
 			synchronized (this.setLastReferenceValues) {
-				setLastReferenceValues.computeIfPresent(accountData.getAddress(), mergePublicKey);
+				setLastReferenceValues.merge(accountData.getAddress(), newPair, mergePublicKey);
 			}
 		}
 
@@ -53,6 +97,12 @@ public class AccountRefCache implements AutoCloseable {
 
 	private Repository repository;
 
+	/**
+	 * Constructs a new account reference cache, unique to passed <tt>repository</tt> handle.
+	 * 
+	 * @param repository
+	 * @throws IllegalStateException if a cache already exists for <tt>repository</tt>
+	 */
 	public AccountRefCache(Repository repository) {
 		RefCache refCache = new RefCache();
 
@@ -64,9 +114,17 @@ public class AccountRefCache implements AutoCloseable {
 		this.repository = repository;
 	}
 
+	/**
+	 * Save all cached setLastReference account-reference values into repository.
+	 * <p>
+	 * Closes cache to prevent any future setLastReference() attempts post-commit.
+	 * 
+	 * @throws DataException
+	 */
 	public void commit() throws DataException {
 		RefCache refCache;
 
+		// Also duplicated in close(), this prevents future setLastReference() attempts post-commit.
 		synchronized (CACHE) {
 			refCache = CACHE.remove(this.repository);
 		}
@@ -89,12 +147,29 @@ public class AccountRefCache implements AutoCloseable {
 	}
 
 	@Override
-	public void close() throws Exception {
+	public void close() {
 		synchronized (CACHE) {
 			CACHE.remove(this.repository);
 		}
 	}
 
+	/**
+	 * Returns lastReference value for account.
+	 * <p>
+	 * If cache is not in effect for passed <tt>repository</tt> handle,
+	 * then this method fetches lastReference directly from repository.
+	 * <p>
+	 * If cache <i>is</i> in effect, then this method returns cached
+	 * lastReference, which is <b>not</b> affected by calls to
+	 * <tt>setLastReference</tt>.
+	 * <p>
+	 * Typically called by corresponding method in Account class.
+	 * 
+	 * @param repository
+	 * @param address account's address
+	 * @return account's lastReference, or null if account unknown, or lastReference not set
+	 * @throws DataException
+	 */
 	/*package*/ static byte[] getLastReference(Repository repository, String address) throws DataException {
 		RefCache refCache;
 
@@ -108,6 +183,22 @@ public class AccountRefCache implements AutoCloseable {
 		return refCache.getLastReference(repository, address);
 	}
 
+	/**
+	 * Sets lastReference value for account.
+	 * <p>
+	 * If cache is not in effect for passed <tt>repository</tt> handle,
+	 * then this method sets lastReference directly in repository.
+	 * <p>
+	 * If cache <i>is</i> in effect, then this method caches the new
+	 * lastReference, which is <b>not</b> returned by calls to
+	 * <tt>getLastReference</tt>.
+	 * <p>
+	 * Typically called by corresponding method in Account class.
+	 * 
+	 * @param repository
+	 * @param accountData
+	 * @throws DataException
+	 */
 	/*package*/ static void setLastReference(Repository repository, AccountData accountData) throws DataException {
 		RefCache refCache;
 
@@ -115,8 +206,10 @@ public class AccountRefCache implements AutoCloseable {
 			refCache = CACHE.get(repository);
 		}
 
-		if (refCache == null)
+		if (refCache == null) {
 			repository.getAccountRepository().setLastReference(accountData);
+			return;
+		}
 
 		refCache.setLastReference(accountData);
 	}

src/main/java/org/qortal/block/Block.java

@@ -18,6 +18,7 @@ import java.util.stream.Collectors;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.account.Account;
+import org.qortal.account.AccountRefCache;
 import org.qortal.account.PrivateKeyAccount;
 import org.qortal.account.PublicKeyAccount;
 import org.qortal.asset.Asset;
@@ -1015,7 +1016,9 @@ public class Block {
 
 	/** Returns whether block's transactions are valid. */
 	private ValidationResult areTransactionsValid() throws DataException {
-		try {
+		// We're about to (test-)process a batch of transactions,
+		// so create an account reference cache so get/set correct last-references.
+		try (AccountRefCache accountRefCache = new AccountRefCache(repository)) {
 			// Create repository savepoint here so we can rollback to it after testing transactions
 			repository.setSavepoint();
 
@@ -1229,6 +1232,9 @@ public class Block {
 			rewardTransactionFees();
 		}
 
+		// We're about to (test-)process a batch of transactions,
+		// so create an account reference cache so get/set correct last-references.
+		try (AccountRefCache accountRefCache = new AccountRefCache(this.repository)) {
 			// Process transactions (we'll link them to this block after saving the block itself)
 			processTransactions();
 
@@ -1238,6 +1244,10 @@ public class Block {
 			// Process AT fees and save AT states into repository
 			processAtFeesAndStates();
 
+			// Commit new accounts' last-reference changes
+			accountRefCache.commit();
+		}
+
 		// Link block into blockchain by fetching signature of highest block and setting that as our reference
 		BlockData latestBlockData = this.repository.getBlockRepository().fromHeight(blockchainHeight);
 		if (latestBlockData != null)

src/test/java/org/qortal/test/AccountRefCacheTests.java

@@ -1,76 +1,297 @@
 package org.qortal.test;
 
-public class AccountRefCacheTests {
+import static org.junit.Assert.*;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Random;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.qortal.account.Account;
+import org.qortal.account.AccountRefCache;
+import org.qortal.account.PublicKeyAccount;
+import org.qortal.data.transaction.PaymentTransactionData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.test.common.BlockUtils;
+import org.qortal.test.common.Common;
+import org.qortal.test.common.TestAccount;
+import org.qortal.test.common.TransactionUtils;
+import org.qortal.test.common.transaction.TestTransaction;
+
+public class AccountRefCacheTests extends Common {
+
+	private static final Random RANDOM = new Random();
+
+	@Before
+	public void before() throws DataException {
+		Common.useDefaultSettings();
+	}
+
+	// Test no cache in play (existing account)
+	@Test
+	public void testNoCacheExistingAccount() throws DataException {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			TestAccount account = Common.getTestAccount(repository, "alice");
 
-	// Test no cache in play (existing account):
 			// fetch 1st ref
+			byte[] lastRef1 = account.getLastReference();
+
 			// generate 2nd ref and call Account.setLastReference
-	// fetch 3rd ref
+			byte[] lastRef2 = new byte[32];
-	// 3rd ref should match 2st ref
+			RANDOM.nextBytes(lastRef2);
+			account.setLastReference(lastRef2);
+
+			// fetch 3rd ref
+			byte[] lastRef3 = account.getLastReference();
+
+			// 3rd ref should match 2st ref
+			assertTrue("getLastReference() should return latest value", Arrays.equals(lastRef2, lastRef3));
+
+			// 3rd ref should not match 1st ref
+			assertFalse("setLastReference() failed?", Arrays.equals(lastRef1, lastRef3));
+		}
+	}
+
+	// Test no cache in play (new account)
+	@Test
+	public void testNoCacheNewAccount() throws DataException {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			Account account = createRandomAccount(repository);
 
-	// Test no cache in play (no account):
 			// fetch 1st ref
+			byte[] lastRef1 = account.getLastReference();
+			assertNull("new account's initial lastReference should be null", lastRef1);
+
 			// generate 2nd ref and call Account.setLastReference
+			byte[] lastRef2 = new byte[32];
+			RANDOM.nextBytes(lastRef2);
+			account.setLastReference(lastRef2);
+
 			// fetch 3rd ref
+			byte[] lastRef3 = account.getLastReference();
+
 			// 3rd ref should match 2st ref
+			assertTrue("getLastReference() should return latest value", Arrays.equals(lastRef2, lastRef3));
+
+			// 3rd ref should not match 1st ref
+			assertFalse("setLastReference() failed?", Arrays.equals(lastRef1, lastRef3));
+		}
+	}
+
+	// Test cache in play (existing account, no commit)
+	@Test
+	public void testWithCacheExistingAccountNoCommit() throws DataException {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			TestAccount account = Common.getTestAccount(repository, "alice");
 
-	// Test cache in play (existing account, no commit):
 			// fetch 1st ref
-	// begin caching
+			byte[] lastRef1 = account.getLastReference();
-	// fetch 2nd ref
+
-	// 2nd ref should match 1st ref
+			// begin caching
-	// generate 3rd ref and call Account.setLastReference
+			try (final AccountRefCache accountRefCache = new AccountRefCache(repository)) {
-	// fetch 4th ref
+				// fetch 2nd ref
-	// 4th ref should match 1st ref
+				byte[] lastRef2 = account.getLastReference();
-	// discard cache
+
-	// fetch 5th ref
+				// 2nd ref should match 1st ref
-	// 5th ref should match 1st ref
+				assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef2));
+
+				// generate 3rd ref and call Account.setLastReference
+				byte[] lastRef3 = new byte[32];
+				RANDOM.nextBytes(lastRef3);
+				account.setLastReference(lastRef3);
+
+				// fetch 4th ref
+				byte[] lastRef4 = account.getLastReference();
+
+				// 4th ref should match 1st ref
+				assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef4));
+			}
+			// cache discarded
+
+			// fetch 5th ref
+			byte[] lastRef5 = account.getLastReference();
+
+			// 5th ref should match 1st ref
+			assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef5));
+		}
+	}
+
+	// Test cache in play (existing account, with commit)
+	@Test
+	public void testWithCacheExistingAccountWithCommit() throws DataException {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			TestAccount account = Common.getTestAccount(repository, "alice");
 
-	// Test cache in play (existing account, with commit):
 			// fetch 1st ref
-	// begin caching
+			byte[] lastRef1 = account.getLastReference();
-	// fetch 2nd ref
-	// 2nd ref should match 1st ref
-	// generate 3rd ref and call Account.setLastReference
-	// fetch 4th ref
-	// 4th ref should match 1st ref
-	// commit cache
-	// fetch 5th ref
-	// 5th ref should match 3rd ref
 
-	// Test cache in play (new account, no commit):
-	// fetch 1st ref (null)
 			// begin caching
+			byte[] committedRef;
+			try (final AccountRefCache accountRefCache = new AccountRefCache(repository)) {
 				// fetch 2nd ref
+				byte[] lastRef2 = account.getLastReference();
+
 				// 2nd ref should match 1st ref
+				assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef2));
+
 				// generate 3rd ref and call Account.setLastReference
+				byte[] lastRef3 = new byte[32];
+				RANDOM.nextBytes(lastRef3);
+				account.setLastReference(lastRef3);
+				committedRef = lastRef3;
+
 				// fetch 4th ref
+				byte[] lastRef4 = account.getLastReference();
+
 				// 4th ref should match 1st ref
-	// discard cache
+				assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef4));
+
+				// Commit cache
+				accountRefCache.commit();
+			}
+
 			// fetch 5th ref
+			byte[] lastRef5 = account.getLastReference();
+
+			// 5th ref should match committed ref
+			assertTrue("getLastReference() should return pre-cache value", Arrays.equals(committedRef, lastRef5));
+		}
+	}
+
+	// Test cache in play (new account, no commit)
+	@Test
+	public void testWithCacheNewAccountNoCommit() throws DataException {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			Account account = createRandomAccount(repository);
+
+			// fetch 1st ref
+			byte[] lastRef1 = account.getLastReference();
+			assertNull("new account's initial lastReference should be null", lastRef1);
+
+			// begin caching
+			try (final AccountRefCache accountRefCache = new AccountRefCache(repository)) {
+				// fetch 2nd ref
+				byte[] lastRef2 = account.getLastReference();
+
+				// 2nd ref should match 1st ref
+				assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef2));
+
+				// generate 3rd ref and call Account.setLastReference
+				byte[] lastRef3 = new byte[32];
+				RANDOM.nextBytes(lastRef3);
+				account.setLastReference(lastRef3);
+
+				// fetch 4th ref
+				byte[] lastRef4 = account.getLastReference();
+
+				// 4th ref should match 1st ref
+				assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef4));
+			}
+			// cache discarded
+
+			// fetch 5th ref
+			byte[] lastRef5 = account.getLastReference();
+
 			// 5th ref should match 1st ref
+			assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef5));
+		}
+	}
+
+	// Test cache in play (new account, with commit)
+	@Test
+	public void testWithCacheNewAccountWithCommit() throws DataException {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			Account account = createRandomAccount(repository);
+
+			// fetch 1st ref
+			byte[] lastRef1 = account.getLastReference();
+			assertNull("new account's initial lastReference should be null", lastRef1);
 
-	// Test cache in play (new account, with commit):
-	// fetch 1st ref (null)
 			// begin caching
+			byte[] committedRef;
+			try (final AccountRefCache accountRefCache = new AccountRefCache(repository)) {
 				// fetch 2nd ref
+				byte[] lastRef2 = account.getLastReference();
+
 				// 2nd ref should match 1st ref
+				assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef2));
+
 				// generate 3rd ref and call Account.setLastReference
+				byte[] lastRef3 = new byte[32];
+				RANDOM.nextBytes(lastRef3);
+				account.setLastReference(lastRef3);
+				committedRef = lastRef3;
+
 				// fetch 4th ref
+				byte[] lastRef4 = account.getLastReference();
+
 				// 4th ref should match 1st ref
-	// commit cache
+				assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef4));
+
+				// Commit cache
+				accountRefCache.commit();
+			}
+
 			// fetch 5th ref
-	// 5th ref should match 3rd ref
+			byte[] lastRef5 = account.getLastReference();
+
+			// 5th ref should match committed ref
+			assertTrue("getLastReference() should return pre-cache value", Arrays.equals(committedRef, lastRef5));
+		}
+	}
 
 	// Test Block support
-	// fetch 1st ref for Alice
+	@Test
-	// generate new payment from Alice to new account Ellen
+	public void testBlockSupport() throws DataException {
-	// generate another payment from Alice to new account Ellen
+		final BigDecimal amount = BigDecimal.valueOf(12345670000L, 8);
-	// mint block containing payments
+
-	// confirm Ellen's ref is 1st payment's sig
+		try (final Repository repository = RepositoryManager.getRepository()) {
-	// confirm Alice's ref if 2nd payment's sig
+			TestAccount alice = Common.getTestAccount(repository, "alice");
+			Account newbie = createRandomAccount(repository);
+
+			// fetch 1st ref
+			byte[] lastRef1 = alice.getLastReference();
+
+			// generate new payment from Alice to new account
+			TransactionData paymentData1 = new PaymentTransactionData(TestTransaction.generateBase(alice), newbie.getAddress(), amount);
+			TransactionUtils.signAsUnconfirmed(repository, paymentData1, alice); // updates paymentData1's signature
+
+			// generate another payment from Alice to new account
+			TransactionData paymentData2 = new PaymentTransactionData(TestTransaction.generateBase(alice), newbie.getAddress(), amount);
+			TransactionUtils.signAsUnconfirmed(repository, paymentData2, alice); // updates paymentData2's signature
+
+			// mint block containing payments (uses cache)
+			BlockUtils.mintBlock(repository);
+
+			// confirm new account's ref is last payment's sig
+			byte[] newAccountRef = newbie.getLastReference();
+			assertTrue("new account's lastReference should match last payment's sig", Arrays.equals(paymentData2.getSignature(), newAccountRef));
+
+			// confirm Alice's ref is last payment's sig
+			byte[] lastRef2 = alice.getLastReference();
+			assertTrue("Alice's lastReference should match last payment's sig", Arrays.equals(paymentData2.getSignature(), lastRef2));
+
 			// orphan block
-	// confirm Ellen's ref is null
+			BlockUtils.orphanLastBlock(repository);
+
+			// confirm new account's ref reverted back to null
+			newAccountRef = newbie.getLastReference();
+			assertNull("new account's lastReference should have reverted back to null", newAccountRef);
+
 			// confirm Alice's ref matches 1st ref
+			byte[] lastRef3 = alice.getLastReference();
+			assertTrue("Alice's lastReference should match initial lastReference", Arrays.equals(lastRef1, lastRef3));
+		}
+	}
+
+	private static Account createRandomAccount(Repository repository) {
+		byte[] randomPublicKey = new byte[32];
+		RANDOM.nextBytes(randomPublicKey);
+		return new PublicKeyAccount(repository, randomPublicKey);
+	}
 
 }