From 1b036b763c622965965151ff17ae534e51ccb5a4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 5 Mar 2022 16:10:43 +0000 Subject: [PATCH] Major CPU optimization to block minter Load sorted list of reward share public keys into memory, so that the indexes can be obtained. This is around 100x faster than querying each index separately (and the savings will increase as more keys are added). For 4150 reward share keys, it was taking around 5000ms to query individually, vs 50ms using this approach. The main trade off is that these 4150 keys require around 130kB of additional memory when minting (and this will increase proportionally with more minters). However, this one query was often accounting for 50% of the entire core's CPU usage, so the additional memory usage seems insignificant by comparison. To gain confidence, I ran both old and new approaches side by side, and confirmed that the indexes matched exactly. --- src/main/java/org/qortal/block/Block.java | 38 +++++++++++++------ .../qortal/repository/AccountRepository.java | 2 + .../hsqldb/HSQLDBAccountRepository.java | 21 ++++++++++ 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 927249fa..a1532cd8 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -8,13 +8,7 @@ import java.math.BigInteger; import java.math.RoundingMode; import java.text.DecimalFormat; import java.text.NumberFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import org.apache.logging.log4j.Level; @@ -333,6 +327,11 @@ public class Block { onlineAccountsTimestamp = onlineAccountData.getTimestamp(); } + // Load sorted list of reward share public keys into memory, so that the indexes can be obtained. + // This is up to 100x faster than querying each index separately. For 4150 reward share keys, it + // was taking around 5000ms to query individually, vs 50ms using this approach. + List allRewardSharePublicKeys = repository.getAccountRepository().getRewardSharePublicKeys(); + // Map using index into sorted list of reward-shares as key Map indexedOnlineAccounts = new HashMap<>(); for (OnlineAccountData onlineAccountData : onlineAccounts) { @@ -340,10 +339,7 @@ public class Block { if (onlineAccountData.getTimestamp() != onlineAccountsTimestamp) continue; - Integer accountIndex = repository.getAccountRepository().getRewardShareIndex(onlineAccountData.getPublicKey()); - if (accountIndex == null) - // Online account (reward-share) with current timestamp but reward-share cancelled - continue; + Integer accountIndex = getRewardShareIndex(onlineAccountData.getPublicKey(), allRewardSharePublicKeys); indexedOnlineAccounts.put(accountIndex, onlineAccountData); } @@ -2029,6 +2025,26 @@ public class Block { this.repository.getAccountRepository().tidy(); } + // Utils + + /** + * Find index of rewardSharePublicKey in list of rewardSharePublicKeys + * + * @param rewardSharePublicKey - the key to query + * @param rewardSharePublicKeys - a sorted list of keys + * @return - the index of the key, or null if not found + */ + private static Integer getRewardShareIndex(byte[] rewardSharePublicKey, List rewardSharePublicKeys) { + int index = 0; + for (byte[] publicKey : rewardSharePublicKeys) { + if (Arrays.equals(rewardSharePublicKey, publicKey)) { + return index; + } + index++; + } + return null; + } + private void logDebugInfo() { try { // Avoid calculations if possible. We have to check against INFO here, since Level.isMoreSpecificThan() confusingly uses <= rather than just < diff --git a/src/main/java/org/qortal/repository/AccountRepository.java b/src/main/java/org/qortal/repository/AccountRepository.java index 256f9556..e4c53e9b 100644 --- a/src/main/java/org/qortal/repository/AccountRepository.java +++ b/src/main/java/org/qortal/repository/AccountRepository.java @@ -149,6 +149,8 @@ public interface AccountRepository { public RewardShareData getRewardShare(byte[] rewardSharePublicKey) throws DataException; + public List getRewardSharePublicKeys() throws DataException; + public boolean isRewardSharePublicKey(byte[] publicKey) throws DataException; /** Returns number of active reward-shares involving passed public key as the minting account only. */ diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java index b28a224c..ca1b73cd 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java @@ -633,6 +633,27 @@ public class HSQLDBAccountRepository implements AccountRepository { } } + @Override + public List getRewardSharePublicKeys() throws DataException { + String sql = "SELECT reward_share_public_key FROM RewardShares ORDER BY reward_share_public_key"; + + List rewardSharePublicKeys = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return null; + + do { + byte[] rewardSharePublicKey = resultSet.getBytes(1); + rewardSharePublicKeys.add(rewardSharePublicKey); + } while (resultSet.next()); + + return rewardSharePublicKeys; + } catch (SQLException e) { + throw new DataException("Unable to fetch reward-share public keys from repository", e); + } + } + @Override public boolean isRewardSharePublicKey(byte[] publicKey) throws DataException { try {