Account lastReference cache, now with Block support.

As this changes how lastReferences are checked and updated,
this is not suitable for rolling into current chain without a
"feature trigger", or chain restart!

Added unit tests.
This commit is contained in:
catbref 2020-04-27 11:19:51 +01:00
parent e141e98ecc
commit 0006911e0a
3 changed files with 406 additions and 82 deletions

View File

@ -3,20 +3,66 @@ package org.qortal.account;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.function.BiFunction;
import java.util.function.BinaryOperator;
import org.qortal.data.account.AccountData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.utils.Pair;
/**
* Account lastReference caching
* <p>
* When checking an account's lastReference, the value returned should be the
* most recent value set after processing the most recent block.
* <p>
* However, when processing a batch of transactions, e.g. during block processing or validation,
* each transaction needs to check, and maybe update, multiple accounts' lastReference values.
* <p>
* Because the intermediate updates would affect future checks, we set up a cache of that
* maintains a consistent value for fetching lastReference, but also tracks the latest new
* value, without the overhead of repository calls.
* <p>
* Thus, when batch transaction processing is finished, only the latest new lastReference values
* can be committed to the repository, via {@link AccountRefCache#commit()}.
* <p>
* Getting and setting lastReferences values are done the usual way via
* {@link Account#getLastReference()} and {@link Account#setLastReference(byte[])} which call
* package-visibility methods in <tt>AccountRefCache</tt>.
* <p>
* If {@link Account#getLastReference()} or {@link Account#setLastReference(byte[])} are called
* outside of caching then lastReference values are fetched/set directly from/to the repository.
* <p>
* <tt>AccountRefCache</tt> implements <tt>AutoCloseable</tt> for (typical) use in a try-with-resources block.
*
* @see Account#getLastReference()
* @see Account#setLastReference(byte[])
* @see org.qortal.block.Block#process()
*/
public class AccountRefCache implements AutoCloseable {
private static final Map<Repository, RefCache> CACHE = new HashMap<Repository, RefCache>();
private static final Map<Repository, RefCache> CACHE = new HashMap<>();
private static class RefCache {
private final Map<String, byte[]> getLastReferenceValues = new HashMap<String, byte[]>();
private final Map<String, Pair<byte[], byte[]>> setLastReferenceValues = new HashMap<String, Pair<byte[], byte[]>>();
private final Map<String, byte[]> getLastReferenceValues = new HashMap<>();
private final Map<String, Pair<byte[], byte[]>> setLastReferenceValues = new HashMap<>();
/**
* Function for merging publicKey from new data with old publicKey from map.
* <p>
* Last reference is <tt>A</tt> element in pair.<br>
* Public key is <tt>B</tt> element in pair.
*/
private static final BinaryOperator<Pair<byte[], byte[]>> mergePublicKey = (oldPair, newPair) -> {
// If passed new pair contains non-null publicKey, then we use that one in preference.
if (newPair.getB() == null)
// Otherwise, inherit publicKey from old map value.
newPair.setB(oldPair.getB());
// We always use new lastReference from new pair.
return newPair;
};
public byte[] getLastReference(Repository repository, String address) throws DataException {
synchronized (this.getLastReferenceValues) {
@ -36,13 +82,11 @@ public class AccountRefCache implements AutoCloseable {
}
public void setLastReference(AccountData accountData) {
BiFunction<String, Pair<byte[], byte[]>, Pair<byte[], byte[]>> mergePublicKey = (key, oldPair) -> {
byte[] mergedPublicKey = accountData.getPublicKey() != null ? accountData.getPublicKey() : oldPair.getB();
return new Pair<>(accountData.getReference(), mergedPublicKey);
};
// We're only interested in lastReference and publicKey
Pair<byte[], byte[]> newPair = new Pair<>(accountData.getReference(), accountData.getPublicKey());
synchronized (this.setLastReferenceValues) {
setLastReferenceValues.computeIfPresent(accountData.getAddress(), mergePublicKey);
setLastReferenceValues.merge(accountData.getAddress(), newPair, mergePublicKey);
}
}
@ -53,6 +97,12 @@ public class AccountRefCache implements AutoCloseable {
private Repository repository;
/**
* Constructs a new account reference cache, unique to passed <tt>repository</tt> handle.
*
* @param repository
* @throws IllegalStateException if a cache already exists for <tt>repository</tt>
*/
public AccountRefCache(Repository repository) {
RefCache refCache = new RefCache();
@ -64,9 +114,17 @@ public class AccountRefCache implements AutoCloseable {
this.repository = repository;
}
/**
* Save all cached setLastReference account-reference values into repository.
* <p>
* Closes cache to prevent any future setLastReference() attempts post-commit.
*
* @throws DataException
*/
public void commit() throws DataException {
RefCache refCache;
// Also duplicated in close(), this prevents future setLastReference() attempts post-commit.
synchronized (CACHE) {
refCache = CACHE.remove(this.repository);
}
@ -89,12 +147,29 @@ public class AccountRefCache implements AutoCloseable {
}
@Override
public void close() throws Exception {
public void close() {
synchronized (CACHE) {
CACHE.remove(this.repository);
}
}
/**
* Returns lastReference value for account.
* <p>
* If cache is not in effect for passed <tt>repository</tt> handle,
* then this method fetches lastReference directly from repository.
* <p>
* If cache <i>is</i> in effect, then this method returns cached
* lastReference, which is <b>not</b> affected by calls to
* <tt>setLastReference</tt>.
* <p>
* Typically called by corresponding method in Account class.
*
* @param repository
* @param address account's address
* @return account's lastReference, or null if account unknown, or lastReference not set
* @throws DataException
*/
/*package*/ static byte[] getLastReference(Repository repository, String address) throws DataException {
RefCache refCache;
@ -108,6 +183,22 @@ public class AccountRefCache implements AutoCloseable {
return refCache.getLastReference(repository, address);
}
/**
* Sets lastReference value for account.
* <p>
* If cache is not in effect for passed <tt>repository</tt> handle,
* then this method sets lastReference directly in repository.
* <p>
* If cache <i>is</i> in effect, then this method caches the new
* lastReference, which is <b>not</b> returned by calls to
* <tt>getLastReference</tt>.
* <p>
* Typically called by corresponding method in Account class.
*
* @param repository
* @param accountData
* @throws DataException
*/
/*package*/ static void setLastReference(Repository repository, AccountData accountData) throws DataException {
RefCache refCache;
@ -115,8 +206,10 @@ public class AccountRefCache implements AutoCloseable {
refCache = CACHE.get(repository);
}
if (refCache == null)
if (refCache == null) {
repository.getAccountRepository().setLastReference(accountData);
return;
}
refCache.setLastReference(accountData);
}

View File

@ -18,6 +18,7 @@ import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.account.Account;
import org.qortal.account.AccountRefCache;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.account.PublicKeyAccount;
import org.qortal.asset.Asset;
@ -1015,7 +1016,9 @@ public class Block {
/** Returns whether block's transactions are valid. */
private ValidationResult areTransactionsValid() throws DataException {
try {
// We're about to (test-)process a batch of transactions,
// so create an account reference cache so get/set correct last-references.
try (AccountRefCache accountRefCache = new AccountRefCache(repository)) {
// Create repository savepoint here so we can rollback to it after testing transactions
repository.setSavepoint();
@ -1229,14 +1232,21 @@ public class Block {
rewardTransactionFees();
}
// Process transactions (we'll link them to this block after saving the block itself)
processTransactions();
// We're about to (test-)process a batch of transactions,
// so create an account reference cache so get/set correct last-references.
try (AccountRefCache accountRefCache = new AccountRefCache(this.repository)) {
// Process transactions (we'll link them to this block after saving the block itself)
processTransactions();
// Group-approval transactions
processGroupApprovalTransactions();
// Group-approval transactions
processGroupApprovalTransactions();
// Process AT fees and save AT states into repository
processAtFeesAndStates();
// Process AT fees and save AT states into repository
processAtFeesAndStates();
// Commit new accounts' last-reference changes
accountRefCache.commit();
}
// Link block into blockchain by fetching signature of highest block and setting that as our reference
BlockData latestBlockData = this.repository.getBlockRepository().fromHeight(blockchainHeight);

View File

@ -1,76 +1,297 @@
package org.qortal.test;
public class AccountRefCacheTests {
import static org.junit.Assert.*;
// Test no cache in play (existing account):
// fetch 1st ref
// generate 2nd ref and call Account.setLastReference
// fetch 3rd ref
// 3rd ref should match 2st ref
import java.math.BigDecimal;
import java.util.Arrays;
import java.util.Random;
// Test no cache in play (no account):
// fetch 1st ref
// generate 2nd ref and call Account.setLastReference
// fetch 3rd ref
// 3rd ref should match 2st ref
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.Account;
import org.qortal.account.AccountRefCache;
import org.qortal.account.PublicKeyAccount;
import org.qortal.data.transaction.PaymentTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TestAccount;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
// Test cache in play (existing account, no commit):
// fetch 1st ref
// begin caching
// fetch 2nd ref
// 2nd ref should match 1st ref
// generate 3rd ref and call Account.setLastReference
// fetch 4th ref
// 4th ref should match 1st ref
// discard cache
// fetch 5th ref
// 5th ref should match 1st ref
public class AccountRefCacheTests extends Common {
// Test cache in play (existing account, with commit):
// fetch 1st ref
// begin caching
// fetch 2nd ref
// 2nd ref should match 1st ref
// generate 3rd ref and call Account.setLastReference
// fetch 4th ref
// 4th ref should match 1st ref
// commit cache
// fetch 5th ref
// 5th ref should match 3rd ref
private static final Random RANDOM = new Random();
// Test cache in play (new account, no commit):
// fetch 1st ref (null)
// begin caching
// fetch 2nd ref
// 2nd ref should match 1st ref
// generate 3rd ref and call Account.setLastReference
// fetch 4th ref
// 4th ref should match 1st ref
// discard cache
// fetch 5th ref
// 5th ref should match 1st ref
@Before
public void before() throws DataException {
Common.useDefaultSettings();
}
// Test cache in play (new account, with commit):
// fetch 1st ref (null)
// begin caching
// fetch 2nd ref
// 2nd ref should match 1st ref
// generate 3rd ref and call Account.setLastReference
// fetch 4th ref
// 4th ref should match 1st ref
// commit cache
// fetch 5th ref
// 5th ref should match 3rd ref
// Test no cache in play (existing account)
@Test
public void testNoCacheExistingAccount() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
TestAccount account = Common.getTestAccount(repository, "alice");
// fetch 1st ref
byte[] lastRef1 = account.getLastReference();
// generate 2nd ref and call Account.setLastReference
byte[] lastRef2 = new byte[32];
RANDOM.nextBytes(lastRef2);
account.setLastReference(lastRef2);
// fetch 3rd ref
byte[] lastRef3 = account.getLastReference();
// 3rd ref should match 2st ref
assertTrue("getLastReference() should return latest value", Arrays.equals(lastRef2, lastRef3));
// 3rd ref should not match 1st ref
assertFalse("setLastReference() failed?", Arrays.equals(lastRef1, lastRef3));
}
}
// Test no cache in play (new account)
@Test
public void testNoCacheNewAccount() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
Account account = createRandomAccount(repository);
// fetch 1st ref
byte[] lastRef1 = account.getLastReference();
assertNull("new account's initial lastReference should be null", lastRef1);
// generate 2nd ref and call Account.setLastReference
byte[] lastRef2 = new byte[32];
RANDOM.nextBytes(lastRef2);
account.setLastReference(lastRef2);
// fetch 3rd ref
byte[] lastRef3 = account.getLastReference();
// 3rd ref should match 2st ref
assertTrue("getLastReference() should return latest value", Arrays.equals(lastRef2, lastRef3));
// 3rd ref should not match 1st ref
assertFalse("setLastReference() failed?", Arrays.equals(lastRef1, lastRef3));
}
}
// Test cache in play (existing account, no commit)
@Test
public void testWithCacheExistingAccountNoCommit() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
TestAccount account = Common.getTestAccount(repository, "alice");
// fetch 1st ref
byte[] lastRef1 = account.getLastReference();
// begin caching
try (final AccountRefCache accountRefCache = new AccountRefCache(repository)) {
// fetch 2nd ref
byte[] lastRef2 = account.getLastReference();
// 2nd ref should match 1st ref
assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef2));
// generate 3rd ref and call Account.setLastReference
byte[] lastRef3 = new byte[32];
RANDOM.nextBytes(lastRef3);
account.setLastReference(lastRef3);
// fetch 4th ref
byte[] lastRef4 = account.getLastReference();
// 4th ref should match 1st ref
assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef4));
}
// cache discarded
// fetch 5th ref
byte[] lastRef5 = account.getLastReference();
// 5th ref should match 1st ref
assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef5));
}
}
// Test cache in play (existing account, with commit)
@Test
public void testWithCacheExistingAccountWithCommit() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
TestAccount account = Common.getTestAccount(repository, "alice");
// fetch 1st ref
byte[] lastRef1 = account.getLastReference();
// begin caching
byte[] committedRef;
try (final AccountRefCache accountRefCache = new AccountRefCache(repository)) {
// fetch 2nd ref
byte[] lastRef2 = account.getLastReference();
// 2nd ref should match 1st ref
assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef2));
// generate 3rd ref and call Account.setLastReference
byte[] lastRef3 = new byte[32];
RANDOM.nextBytes(lastRef3);
account.setLastReference(lastRef3);
committedRef = lastRef3;
// fetch 4th ref
byte[] lastRef4 = account.getLastReference();
// 4th ref should match 1st ref
assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef4));
// Commit cache
accountRefCache.commit();
}
// fetch 5th ref
byte[] lastRef5 = account.getLastReference();
// 5th ref should match committed ref
assertTrue("getLastReference() should return pre-cache value", Arrays.equals(committedRef, lastRef5));
}
}
// Test cache in play (new account, no commit)
@Test
public void testWithCacheNewAccountNoCommit() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
Account account = createRandomAccount(repository);
// fetch 1st ref
byte[] lastRef1 = account.getLastReference();
assertNull("new account's initial lastReference should be null", lastRef1);
// begin caching
try (final AccountRefCache accountRefCache = new AccountRefCache(repository)) {
// fetch 2nd ref
byte[] lastRef2 = account.getLastReference();
// 2nd ref should match 1st ref
assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef2));
// generate 3rd ref and call Account.setLastReference
byte[] lastRef3 = new byte[32];
RANDOM.nextBytes(lastRef3);
account.setLastReference(lastRef3);
// fetch 4th ref
byte[] lastRef4 = account.getLastReference();
// 4th ref should match 1st ref
assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef4));
}
// cache discarded
// fetch 5th ref
byte[] lastRef5 = account.getLastReference();
// 5th ref should match 1st ref
assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef5));
}
}
// Test cache in play (new account, with commit)
@Test
public void testWithCacheNewAccountWithCommit() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
Account account = createRandomAccount(repository);
// fetch 1st ref
byte[] lastRef1 = account.getLastReference();
assertNull("new account's initial lastReference should be null", lastRef1);
// begin caching
byte[] committedRef;
try (final AccountRefCache accountRefCache = new AccountRefCache(repository)) {
// fetch 2nd ref
byte[] lastRef2 = account.getLastReference();
// 2nd ref should match 1st ref
assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef2));
// generate 3rd ref and call Account.setLastReference
byte[] lastRef3 = new byte[32];
RANDOM.nextBytes(lastRef3);
account.setLastReference(lastRef3);
committedRef = lastRef3;
// fetch 4th ref
byte[] lastRef4 = account.getLastReference();
// 4th ref should match 1st ref
assertTrue("getLastReference() should return pre-cache value", Arrays.equals(lastRef1, lastRef4));
// Commit cache
accountRefCache.commit();
}
// fetch 5th ref
byte[] lastRef5 = account.getLastReference();
// 5th ref should match committed ref
assertTrue("getLastReference() should return pre-cache value", Arrays.equals(committedRef, lastRef5));
}
}
// Test Block support
// fetch 1st ref for Alice
// generate new payment from Alice to new account Ellen
// generate another payment from Alice to new account Ellen
// mint block containing payments
// confirm Ellen's ref is 1st payment's sig
// confirm Alice's ref if 2nd payment's sig
// orphan block
// confirm Ellen's ref is null
// confirm Alice's ref matches 1st ref
@Test
public void testBlockSupport() throws DataException {
final BigDecimal amount = BigDecimal.valueOf(12345670000L, 8);
try (final Repository repository = RepositoryManager.getRepository()) {
TestAccount alice = Common.getTestAccount(repository, "alice");
Account newbie = createRandomAccount(repository);
// fetch 1st ref
byte[] lastRef1 = alice.getLastReference();
// generate new payment from Alice to new account
TransactionData paymentData1 = new PaymentTransactionData(TestTransaction.generateBase(alice), newbie.getAddress(), amount);
TransactionUtils.signAsUnconfirmed(repository, paymentData1, alice); // updates paymentData1's signature
// generate another payment from Alice to new account
TransactionData paymentData2 = new PaymentTransactionData(TestTransaction.generateBase(alice), newbie.getAddress(), amount);
TransactionUtils.signAsUnconfirmed(repository, paymentData2, alice); // updates paymentData2's signature
// mint block containing payments (uses cache)
BlockUtils.mintBlock(repository);
// confirm new account's ref is last payment's sig
byte[] newAccountRef = newbie.getLastReference();
assertTrue("new account's lastReference should match last payment's sig", Arrays.equals(paymentData2.getSignature(), newAccountRef));
// confirm Alice's ref is last payment's sig
byte[] lastRef2 = alice.getLastReference();
assertTrue("Alice's lastReference should match last payment's sig", Arrays.equals(paymentData2.getSignature(), lastRef2));
// orphan block
BlockUtils.orphanLastBlock(repository);
// confirm new account's ref reverted back to null
newAccountRef = newbie.getLastReference();
assertNull("new account's lastReference should have reverted back to null", newAccountRef);
// confirm Alice's ref matches 1st ref
byte[] lastRef3 = alice.getLastReference();
assertTrue("Alice's lastReference should match initial lastReference", Arrays.equals(lastRef1, lastRef3));
}
}
private static Account createRandomAccount(Repository repository) {
byte[] randomPublicKey = new byte[32];
RANDOM.nextBytes(randomPublicKey);
return new PublicKeyAccount(repository, randomPublicKey);
}
}