From 0006911e0a163566715439d624018d6015908de4 Mon Sep 17 00:00:00 2001 From: catbref Date: Mon, 27 Apr 2020 11:19:51 +0100 Subject: [PATCH] Account lastReference cache, now with Block support. As this changes how lastReferences are checked and updated, this is not suitable for rolling into current chain without a "feature trigger", or chain restart! Added unit tests. --- .../org/qortal/account/AccountRefCache.java | 115 +++++- src/main/java/org/qortal/block/Block.java | 24 +- .../org/qortal/test/AccountRefCacheTests.java | 349 ++++++++++++++---- 3 files changed, 406 insertions(+), 82 deletions(-) diff --git a/src/main/java/org/qortal/account/AccountRefCache.java b/src/main/java/org/qortal/account/AccountRefCache.java index 674b8044..13017bd3 100644 --- a/src/main/java/org/qortal/account/AccountRefCache.java +++ b/src/main/java/org/qortal/account/AccountRefCache.java @@ -3,20 +3,66 @@ package org.qortal.account; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; -import java.util.function.BiFunction; +import java.util.function.BinaryOperator; import org.qortal.data.account.AccountData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.utils.Pair; +/** + * Account lastReference caching + *

+ * When checking an account's lastReference, the value returned should be the + * most recent value set after processing the most recent block. + *

+ * However, when processing a batch of transactions, e.g. during block processing or validation, + * each transaction needs to check, and maybe update, multiple accounts' lastReference values. + *

+ * Because the intermediate updates would affect future checks, we set up a cache of that + * maintains a consistent value for fetching lastReference, but also tracks the latest new + * value, without the overhead of repository calls. + *

+ * Thus, when batch transaction processing is finished, only the latest new lastReference values + * can be committed to the repository, via {@link AccountRefCache#commit()}. + *

+ * Getting and setting lastReferences values are done the usual way via + * {@link Account#getLastReference()} and {@link Account#setLastReference(byte[])} which call + * package-visibility methods in AccountRefCache. + *

+ * If {@link Account#getLastReference()} or {@link Account#setLastReference(byte[])} are called + * outside of caching then lastReference values are fetched/set directly from/to the repository. + *

+ * AccountRefCache implements AutoCloseable for (typical) use in a try-with-resources block. + * + * @see Account#getLastReference() + * @see Account#setLastReference(byte[]) + * @see org.qortal.block.Block#process() + */ public class AccountRefCache implements AutoCloseable { - private static final Map CACHE = new HashMap(); + private static final Map CACHE = new HashMap<>(); private static class RefCache { - private final Map getLastReferenceValues = new HashMap(); - private final Map> setLastReferenceValues = new HashMap>(); + private final Map getLastReferenceValues = new HashMap<>(); + private final Map> setLastReferenceValues = new HashMap<>(); + + /** + * Function for merging publicKey from new data with old publicKey from map. + *

+ * Last reference is A element in pair.
+ * Public key is B element in pair. + */ + private static final BinaryOperator> mergePublicKey = (oldPair, newPair) -> { + // If passed new pair contains non-null publicKey, then we use that one in preference. + if (newPair.getB() == null) + // Otherwise, inherit publicKey from old map value. + newPair.setB(oldPair.getB()); + + // We always use new lastReference from new pair. + return newPair; + }; + public byte[] getLastReference(Repository repository, String address) throws DataException { synchronized (this.getLastReferenceValues) { @@ -36,13 +82,11 @@ public class AccountRefCache implements AutoCloseable { } public void setLastReference(AccountData accountData) { - BiFunction, Pair> mergePublicKey = (key, oldPair) -> { - byte[] mergedPublicKey = accountData.getPublicKey() != null ? accountData.getPublicKey() : oldPair.getB(); - return new Pair<>(accountData.getReference(), mergedPublicKey); - }; + // We're only interested in lastReference and publicKey + Pair newPair = new Pair<>(accountData.getReference(), accountData.getPublicKey()); synchronized (this.setLastReferenceValues) { - setLastReferenceValues.computeIfPresent(accountData.getAddress(), mergePublicKey); + setLastReferenceValues.merge(accountData.getAddress(), newPair, mergePublicKey); } } @@ -53,6 +97,12 @@ public class AccountRefCache implements AutoCloseable { private Repository repository; + /** + * Constructs a new account reference cache, unique to passed repository handle. + * + * @param repository + * @throws IllegalStateException if a cache already exists for repository + */ public AccountRefCache(Repository repository) { RefCache refCache = new RefCache(); @@ -64,9 +114,17 @@ public class AccountRefCache implements AutoCloseable { this.repository = repository; } + /** + * Save all cached setLastReference account-reference values into repository. + *

+ * Closes cache to prevent any future setLastReference() attempts post-commit. + * + * @throws DataException + */ public void commit() throws DataException { RefCache refCache; + // Also duplicated in close(), this prevents future setLastReference() attempts post-commit. synchronized (CACHE) { refCache = CACHE.remove(this.repository); } @@ -89,12 +147,29 @@ public class AccountRefCache implements AutoCloseable { } @Override - public void close() throws Exception { + public void close() { synchronized (CACHE) { CACHE.remove(this.repository); } } + /** + * Returns lastReference value for account. + *

+ * If cache is not in effect for passed repository handle, + * then this method fetches lastReference directly from repository. + *

+ * If cache is in effect, then this method returns cached + * lastReference, which is not affected by calls to + * setLastReference. + *

+ * Typically called by corresponding method in Account class. + * + * @param repository + * @param address account's address + * @return account's lastReference, or null if account unknown, or lastReference not set + * @throws DataException + */ /*package*/ static byte[] getLastReference(Repository repository, String address) throws DataException { RefCache refCache; @@ -108,6 +183,22 @@ public class AccountRefCache implements AutoCloseable { return refCache.getLastReference(repository, address); } + /** + * Sets lastReference value for account. + *

+ * If cache is not in effect for passed repository handle, + * then this method sets lastReference directly in repository. + *

+ * If cache is in effect, then this method caches the new + * lastReference, which is not returned by calls to + * getLastReference. + *

+ * Typically called by corresponding method in Account class. + * + * @param repository + * @param accountData + * @throws DataException + */ /*package*/ static void setLastReference(Repository repository, AccountData accountData) throws DataException { RefCache refCache; @@ -115,8 +206,10 @@ public class AccountRefCache implements AutoCloseable { refCache = CACHE.get(repository); } - if (refCache == null) + if (refCache == null) { repository.getAccountRepository().setLastReference(accountData); + return; + } refCache.setLastReference(accountData); } diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 693e1bd0..f5d06e1a 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -18,6 +18,7 @@ import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.account.Account; +import org.qortal.account.AccountRefCache; import org.qortal.account.PrivateKeyAccount; import org.qortal.account.PublicKeyAccount; import org.qortal.asset.Asset; @@ -1015,7 +1016,9 @@ public class Block { /** Returns whether block's transactions are valid. */ private ValidationResult areTransactionsValid() throws DataException { - try { + // We're about to (test-)process a batch of transactions, + // so create an account reference cache so get/set correct last-references. + try (AccountRefCache accountRefCache = new AccountRefCache(repository)) { // Create repository savepoint here so we can rollback to it after testing transactions repository.setSavepoint(); @@ -1229,14 +1232,21 @@ public class Block { rewardTransactionFees(); } - // Process transactions (we'll link them to this block after saving the block itself) - processTransactions(); + // We're about to (test-)process a batch of transactions, + // so create an account reference cache so get/set correct last-references. + try (AccountRefCache accountRefCache = new AccountRefCache(this.repository)) { + // Process transactions (we'll link them to this block after saving the block itself) + processTransactions(); - // Group-approval transactions - processGroupApprovalTransactions(); + // Group-approval transactions + processGroupApprovalTransactions(); - // Process AT fees and save AT states into repository - processAtFeesAndStates(); + // Process AT fees and save AT states into repository + processAtFeesAndStates(); + + // Commit new accounts' last-reference changes + accountRefCache.commit(); + } // Link block into blockchain by fetching signature of highest block and setting that as our reference BlockData latestBlockData = this.repository.getBlockRepository().fromHeight(blockchainHeight); diff --git a/src/test/java/org/qortal/test/AccountRefCacheTests.java b/src/test/java/org/qortal/test/AccountRefCacheTests.java index a070bb5a..4ccd8016 100644 --- a/src/test/java/org/qortal/test/AccountRefCacheTests.java +++ b/src/test/java/org/qortal/test/AccountRefCacheTests.java @@ -1,76 +1,297 @@ package org.qortal.test; -public class AccountRefCacheTests { +import static org.junit.Assert.*; - // Test no cache in play (existing account): - // fetch 1st ref - // generate 2nd ref and call Account.setLastReference - // fetch 3rd ref - // 3rd ref should match 2st ref +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.Random; - // Test no cache in play (no account): - // fetch 1st ref - // generate 2nd ref and call Account.setLastReference - // fetch 3rd ref - // 3rd ref should match 2st ref +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.Account; +import org.qortal.account.AccountRefCache; +import org.qortal.account.PublicKeyAccount; +import org.qortal.data.transaction.PaymentTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TestAccount; +import org.qortal.test.common.TransactionUtils; +import org.qortal.test.common.transaction.TestTransaction; - // Test cache in play (existing account, no commit): - // fetch 1st ref - // begin caching - // fetch 2nd ref - // 2nd ref should match 1st ref - // generate 3rd ref and call Account.setLastReference - // fetch 4th ref - // 4th ref should match 1st ref - // discard cache - // fetch 5th ref - // 5th ref should match 1st ref +public class AccountRefCacheTests extends Common { - // Test cache in play (existing account, with commit): - // fetch 1st ref - // begin caching - // fetch 2nd ref - // 2nd ref should match 1st ref - // generate 3rd ref and call Account.setLastReference - // fetch 4th ref - // 4th ref should match 1st ref - // commit cache - // fetch 5th ref - // 5th ref should match 3rd ref + private static final Random RANDOM = new Random(); - // Test cache in play (new account, no commit): - // fetch 1st ref (null) - // begin caching - // fetch 2nd ref - // 2nd ref should match 1st ref - // generate 3rd ref and call Account.setLastReference - // fetch 4th ref - // 4th ref should match 1st ref - // discard cache - // fetch 5th ref - // 5th ref should match 1st ref + @Before + public void before() throws DataException { + Common.useDefaultSettings(); + } - // Test cache in play (new account, with commit): - // fetch 1st ref (null) - // begin caching - // fetch 2nd ref - // 2nd ref should match 1st ref - // generate 3rd ref and call Account.setLastReference - // fetch 4th ref - // 4th ref should match 1st ref - // commit cache - // fetch 5th ref - // 5th ref should match 3rd ref + // Test no cache in play (existing account) + @Test + public void testNoCacheExistingAccount() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + TestAccount account = Common.getTestAccount(repository, "alice"); + + // fetch 1st ref + byte[] lastRef1 = account.getLastReference(); + + // generate 2nd ref and call Account.setLastReference + byte[] lastRef2 = new byte[32]; + RANDOM.nextBytes(lastRef2); + account.setLastReference(lastRef2); + + // fetch 3rd ref + byte[] lastRef3 = account.getLastReference(); + + // 3rd ref should match 2st ref + assertTrue("getLastReference() should return latest value", Arrays.equals(lastRef2, lastRef3)); + + // 3rd ref should not match 1st ref + assertFalse("setLastReference() failed?", Arrays.equals(lastRef1, lastRef3)); + } + } + + // Test no cache in play (new account) + @Test + public void testNoCacheNewAccount() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + Account account = createRandomAccount(repository); + + // fetch 1st ref + byte[] lastRef1 = account.getLastReference(); + assertNull("new account's initial lastReference should be null", lastRef1); + + // generate 2nd ref and call Account.setLastReference + byte[] lastRef2 = new byte[32]; + RANDOM.nextBytes(lastRef2); + account.setLastReference(lastRef2); + + // fetch 3rd ref + byte[] lastRef3 = account.getLastReference(); + + // 3rd ref should match 2st ref + assertTrue("getLastReference() should return latest value", Arrays.equals(lastRef2, lastRef3)); + + // 3rd ref should not match 1st ref + assertFalse("setLastReference() failed?", Arrays.equals(lastRef1, lastRef3)); + } + } + + // Test cache in play (existing account, no commit) + @Test + public void testWithCacheExistingAccountNoCommit() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + TestAccount account = Common.getTestAccount(repository, "alice"); + + // fetch 1st ref + byte[] lastRef1 = account.getLastReference(); + + // begin caching + try (final AccountRefCache accountRefCache = new AccountRefCache(repository)) { + // fetch 2nd ref + byte[] lastRef2 = account.getLastReference(); + + // 2nd ref should match 1st ref + assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef2)); + + // generate 3rd ref and call Account.setLastReference + byte[] lastRef3 = new byte[32]; + RANDOM.nextBytes(lastRef3); + account.setLastReference(lastRef3); + + // fetch 4th ref + byte[] lastRef4 = account.getLastReference(); + + // 4th ref should match 1st ref + assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef4)); + } + // cache discarded + + // fetch 5th ref + byte[] lastRef5 = account.getLastReference(); + + // 5th ref should match 1st ref + assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef5)); + } + } + + // Test cache in play (existing account, with commit) + @Test + public void testWithCacheExistingAccountWithCommit() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + TestAccount account = Common.getTestAccount(repository, "alice"); + + // fetch 1st ref + byte[] lastRef1 = account.getLastReference(); + + // begin caching + byte[] committedRef; + try (final AccountRefCache accountRefCache = new AccountRefCache(repository)) { + // fetch 2nd ref + byte[] lastRef2 = account.getLastReference(); + + // 2nd ref should match 1st ref + assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef2)); + + // generate 3rd ref and call Account.setLastReference + byte[] lastRef3 = new byte[32]; + RANDOM.nextBytes(lastRef3); + account.setLastReference(lastRef3); + committedRef = lastRef3; + + // fetch 4th ref + byte[] lastRef4 = account.getLastReference(); + + // 4th ref should match 1st ref + assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef4)); + + // Commit cache + accountRefCache.commit(); + } + + // fetch 5th ref + byte[] lastRef5 = account.getLastReference(); + + // 5th ref should match committed ref + assertTrue("getLastReference() should return pre-cache value", Arrays.equals(committedRef, lastRef5)); + } + } + + // Test cache in play (new account, no commit) + @Test + public void testWithCacheNewAccountNoCommit() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + Account account = createRandomAccount(repository); + + // fetch 1st ref + byte[] lastRef1 = account.getLastReference(); + assertNull("new account's initial lastReference should be null", lastRef1); + + // begin caching + try (final AccountRefCache accountRefCache = new AccountRefCache(repository)) { + // fetch 2nd ref + byte[] lastRef2 = account.getLastReference(); + + // 2nd ref should match 1st ref + assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef2)); + + // generate 3rd ref and call Account.setLastReference + byte[] lastRef3 = new byte[32]; + RANDOM.nextBytes(lastRef3); + account.setLastReference(lastRef3); + + // fetch 4th ref + byte[] lastRef4 = account.getLastReference(); + + // 4th ref should match 1st ref + assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef4)); + } + // cache discarded + + // fetch 5th ref + byte[] lastRef5 = account.getLastReference(); + + // 5th ref should match 1st ref + assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef5)); + } + } + + // Test cache in play (new account, with commit) + @Test + public void testWithCacheNewAccountWithCommit() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + Account account = createRandomAccount(repository); + + // fetch 1st ref + byte[] lastRef1 = account.getLastReference(); + assertNull("new account's initial lastReference should be null", lastRef1); + + // begin caching + byte[] committedRef; + try (final AccountRefCache accountRefCache = new AccountRefCache(repository)) { + // fetch 2nd ref + byte[] lastRef2 = account.getLastReference(); + + // 2nd ref should match 1st ref + assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef2)); + + // generate 3rd ref and call Account.setLastReference + byte[] lastRef3 = new byte[32]; + RANDOM.nextBytes(lastRef3); + account.setLastReference(lastRef3); + committedRef = lastRef3; + + // fetch 4th ref + byte[] lastRef4 = account.getLastReference(); + + // 4th ref should match 1st ref + assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef4)); + + // Commit cache + accountRefCache.commit(); + } + + // fetch 5th ref + byte[] lastRef5 = account.getLastReference(); + + // 5th ref should match committed ref + assertTrue("getLastReference() should return pre-cache value", Arrays.equals(committedRef, lastRef5)); + } + } // Test Block support - // fetch 1st ref for Alice - // generate new payment from Alice to new account Ellen - // generate another payment from Alice to new account Ellen - // mint block containing payments - // confirm Ellen's ref is 1st payment's sig - // confirm Alice's ref if 2nd payment's sig - // orphan block - // confirm Ellen's ref is null - // confirm Alice's ref matches 1st ref + @Test + public void testBlockSupport() throws DataException { + final BigDecimal amount = BigDecimal.valueOf(12345670000L, 8); + + try (final Repository repository = RepositoryManager.getRepository()) { + TestAccount alice = Common.getTestAccount(repository, "alice"); + Account newbie = createRandomAccount(repository); + + // fetch 1st ref + byte[] lastRef1 = alice.getLastReference(); + + // generate new payment from Alice to new account + TransactionData paymentData1 = new PaymentTransactionData(TestTransaction.generateBase(alice), newbie.getAddress(), amount); + TransactionUtils.signAsUnconfirmed(repository, paymentData1, alice); // updates paymentData1's signature + + // generate another payment from Alice to new account + TransactionData paymentData2 = new PaymentTransactionData(TestTransaction.generateBase(alice), newbie.getAddress(), amount); + TransactionUtils.signAsUnconfirmed(repository, paymentData2, alice); // updates paymentData2's signature + + // mint block containing payments (uses cache) + BlockUtils.mintBlock(repository); + + // confirm new account's ref is last payment's sig + byte[] newAccountRef = newbie.getLastReference(); + assertTrue("new account's lastReference should match last payment's sig", Arrays.equals(paymentData2.getSignature(), newAccountRef)); + + // confirm Alice's ref is last payment's sig + byte[] lastRef2 = alice.getLastReference(); + assertTrue("Alice's lastReference should match last payment's sig", Arrays.equals(paymentData2.getSignature(), lastRef2)); + + // orphan block + BlockUtils.orphanLastBlock(repository); + + // confirm new account's ref reverted back to null + newAccountRef = newbie.getLastReference(); + assertNull("new account's lastReference should have reverted back to null", newAccountRef); + + // confirm Alice's ref matches 1st ref + byte[] lastRef3 = alice.getLastReference(); + assertTrue("Alice's lastReference should match initial lastReference", Arrays.equals(lastRef1, lastRef3)); + } + } + + private static Account createRandomAccount(Repository repository) { + byte[] randomPublicKey = new byte[32]; + RANDOM.nextBytes(randomPublicKey); + return new PublicKeyAccount(repository, randomPublicKey); + } }