From 5e145de52bf889916b60ada1bf19177ce03cb3a3 Mon Sep 17 00:00:00 2001 From: kennycud Date: Thu, 12 Dec 2024 13:46:18 -0800 Subject: [PATCH] Balance Recorder initial implementation. --- .../qortal/api/resource/AssetsResource.java | 122 ++++++ .../hsqldb/HSQLDBBalanceRecorder.java | 66 ++-- .../data/account/AddressAmountData.java | 54 +++ .../qortal/data/account/BlockHeightRange.java | 51 +++ .../BlockHeightRangeAddressAmounts.java | 52 +++ .../repository/hsqldb/HSQLDBCacheUtils.java | 125 ++++-- .../java/org/qortal/settings/Settings.java | 56 ++- .../qortal/utils/BalanceRecorderUtils.java | 119 ++++++ .../test/utils/BalanceRecorderUtilsTests.java | 361 ++++++++++++++++++ 9 files changed, 943 insertions(+), 63 deletions(-) create mode 100644 src/main/java/org/qortal/data/account/AddressAmountData.java create mode 100644 src/main/java/org/qortal/data/account/BlockHeightRange.java create mode 100644 src/main/java/org/qortal/data/account/BlockHeightRangeAddressAmounts.java create mode 100644 src/main/java/org/qortal/utils/BalanceRecorderUtils.java create mode 100644 src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java diff --git a/src/main/java/org/qortal/api/resource/AssetsResource.java b/src/main/java/org/qortal/api/resource/AssetsResource.java index 40e04256..6de01434 100644 --- a/src/main/java/org/qortal/api/resource/AssetsResource.java +++ b/src/main/java/org/qortal/api/resource/AssetsResource.java @@ -16,9 +16,13 @@ import org.qortal.api.model.AggregatedOrder; import org.qortal.api.model.TradeWithOrderInfo; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; import org.qortal.asset.Asset; +import org.qortal.controller.hsqldb.HSQLDBBalanceRecorder; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; +import org.qortal.data.account.AddressAmountData; +import org.qortal.data.account.BlockHeightRange; +import org.qortal.data.account.BlockHeightRangeAddressAmounts; import org.qortal.data.asset.AssetData; import org.qortal.data.asset.OrderData; import org.qortal.data.asset.RecentTradeData; @@ -33,6 +37,7 @@ import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.ValidationResult; import org.qortal.transform.TransformationException; import org.qortal.transform.transaction.*; +import org.qortal.utils.BalanceRecorderUtils; import org.qortal.utils.Base58; import javax.servlet.http.HttpServletRequest; @@ -42,6 +47,7 @@ import javax.ws.rs.core.MediaType; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; @Path("/assets") @@ -179,6 +185,122 @@ public class AssetsResource { } } + @GET + @Path("/balancedynamicranges") + @Operation( + summary = "Get balance dynamic ranges listed.", + description = ".", + responses = { + @ApiResponse( + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = BlockHeightRange.class + ) + ) + ) + ) + } + ) + public List getBalanceDynamicRanges( + @Parameter(ref = "offset") @QueryParam("offset") Integer offset, + @Parameter(ref = "limit") @QueryParam("limit") Integer limit, + @Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) { + + Optional recorder = HSQLDBBalanceRecorder.getInstance(); + + if( recorder.isPresent()) { + return recorder.get().getRanges(offset, limit, reverse); + } + else { + return new ArrayList<>(0); + } + } + + @GET + @Path("/balancedynamicrange/{height}") + @Operation( + summary = "Get balance dynamic range for a given height.", + description = ".", + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + implementation = BlockHeightRange.class + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.INVALID_CRITERIA, ApiError.INVALID_DATA + }) + public BlockHeightRange getBalanceDynamicRange(@PathParam("height") int height) { + + Optional recorder = HSQLDBBalanceRecorder.getInstance(); + + if( recorder.isPresent()) { + Optional range = recorder.get().getRange(height); + + if( range.isPresent() ) { + return range.get(); + } + else { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + } + else { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + } + + @GET + @Path("/balancedynamicamounts/{begin}/{end}") + @Operation( + summary = "Get balance dynamic ranges address amounts listed.", + description = ".", + responses = { + @ApiResponse( + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = AddressAmountData.class + ) + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.INVALID_CRITERIA, ApiError.INVALID_DATA + }) + public List getBalanceDynamicAddressAmounts( + @PathParam("begin") int begin, + @PathParam("end") int end, + @Parameter(ref = "offset") @QueryParam("offset") Integer offset, + @Parameter(ref = "limit") @QueryParam("limit") Integer limit) { + + Optional recorder = HSQLDBBalanceRecorder.getInstance(); + + if( recorder.isPresent()) { + Optional addressAmounts = recorder.get().getAddressAmounts(new BlockHeightRange(begin, end)); + + if( addressAmounts.isPresent() ) { + return addressAmounts.get().getAmounts().stream() + .sorted(BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_COMPARATOR.reversed()) + .skip(offset) + .limit(limit) + .collect(Collectors.toList()); + } + else { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + } + else { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + } + @GET @Path("/openorders/{assetid}/{otherassetid}") @Operation( diff --git a/src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java b/src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java index 7a7009ff..43e7c542 100644 --- a/src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java +++ b/src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java @@ -2,15 +2,19 @@ package org.qortal.controller.hsqldb; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.util.PropertySource; import org.qortal.data.account.AccountBalanceData; +import org.qortal.data.account.BlockHeightRange; +import org.qortal.data.account.BlockHeightRangeAddressAmounts; import org.qortal.repository.hsqldb.HSQLDBCacheUtils; import org.qortal.settings.Settings; +import org.qortal.utils.BalanceRecorderUtils; -import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.stream.Collectors; public class HSQLDBBalanceRecorder extends Thread{ @@ -23,6 +27,8 @@ public class HSQLDBBalanceRecorder extends Thread{ private ConcurrentHashMap> balancesByAddress = new ConcurrentHashMap<>(); + private CopyOnWriteArrayList balanceDynamics = new CopyOnWriteArrayList<>(); + private int priorityRequested; private int frequency; private int capacity; @@ -61,36 +67,52 @@ public class HSQLDBBalanceRecorder extends Thread{ Thread.currentThread().setName("Balance Recorder"); - HSQLDBCacheUtils.startRecordingBalances(this.balancesByHeight, this.balancesByAddress, this.priorityRequested, this.frequency, this.capacity); + HSQLDBCacheUtils.startRecordingBalances(this.balancesByHeight, this.balanceDynamics, this.priorityRequested, this.frequency, this.capacity); } - public List getLatestRecordings(int limit, long offset) { - ArrayList data; + public List getLatestDynamics(int limit, long offset) { - Optional lastHeight = getLastHeight(); + List latest = this.balanceDynamics.stream() + .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.reversed()) + .skip(offset) + .limit(limit) + .collect(Collectors.toList()); - if(lastHeight.isPresent() ) { - List latest = this.balancesByHeight.get(lastHeight.get()); + return latest; + } - if( latest != null ) { - data = new ArrayList<>(latest.size()); - data.addAll( - latest.stream() - .sorted(Comparator.comparingDouble(AccountBalanceData::getBalance).reversed()) - .skip(offset) - .limit(limit) - .collect(Collectors.toList()) - ); - } - else { - data = new ArrayList<>(0); - } + public List getRanges(Integer offset, Integer limit, Boolean reverse) { + + if( reverse ) { + return this.balanceDynamics.stream() + .map(BlockHeightRangeAddressAmounts::getRange) + .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_COMPARATOR.reversed()) + .skip(offset) + .limit(limit) + .collect(Collectors.toList()); } else { - data = new ArrayList<>(0); + return this.balanceDynamics.stream() + .map(BlockHeightRangeAddressAmounts::getRange) + .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_COMPARATOR) + .skip(offset) + .limit(limit) + .collect(Collectors.toList()); } + } - return data; + public Optional getAddressAmounts(BlockHeightRange range) { + + return this.balanceDynamics.stream() + .filter( dynamic -> dynamic.getRange().equals(range)) + .findAny(); + } + + public Optional getRange( int height ) { + return this.balanceDynamics.stream() + .map(BlockHeightRangeAddressAmounts::getRange) + .filter( range -> range.getBegin() < height && range.getEnd() >= height ) + .findAny(); } private Optional getLastHeight() { diff --git a/src/main/java/org/qortal/data/account/AddressAmountData.java b/src/main/java/org/qortal/data/account/AddressAmountData.java new file mode 100644 index 00000000..d5a9f52b --- /dev/null +++ b/src/main/java/org/qortal/data/account/AddressAmountData.java @@ -0,0 +1,54 @@ +package org.qortal.data.account; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import java.util.Objects; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class AddressAmountData { + + private String address; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long amount; + + public AddressAmountData() { + } + + public AddressAmountData(String address, long amount) { + + this.address = address; + this.amount = amount; + } + + public String getAddress() { + return address; + } + + public long getAmount() { + return amount; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AddressAmountData that = (AddressAmountData) o; + return amount == that.amount && Objects.equals(address, that.address); + } + + @Override + public int hashCode() { + return Objects.hash(address, amount); + } + + @Override + public String toString() { + return "AddressAmountData{" + + "address='" + address + '\'' + + ", amount=" + amount + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/account/BlockHeightRange.java b/src/main/java/org/qortal/data/account/BlockHeightRange.java new file mode 100644 index 00000000..fd356cb3 --- /dev/null +++ b/src/main/java/org/qortal/data/account/BlockHeightRange.java @@ -0,0 +1,51 @@ +package org.qortal.data.account; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class BlockHeightRange { + + private int begin; + + private int end; + + public BlockHeightRange() { + } + + public BlockHeightRange(int begin, int end) { + this.begin = begin; + this.end = end; + } + + public int getBegin() { + return begin; + } + + public int getEnd() { + return end; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BlockHeightRange that = (BlockHeightRange) o; + return begin == that.begin && end == that.end; + } + + @Override + public int hashCode() { + return Objects.hash(begin, end); + } + + @Override + public String toString() { + return "BlockHeightRange{" + + "begin=" + begin + + ", end=" + end + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/account/BlockHeightRangeAddressAmounts.java b/src/main/java/org/qortal/data/account/BlockHeightRangeAddressAmounts.java new file mode 100644 index 00000000..3e2b7042 --- /dev/null +++ b/src/main/java/org/qortal/data/account/BlockHeightRangeAddressAmounts.java @@ -0,0 +1,52 @@ +package org.qortal.data.account; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.List; +import java.util.Objects; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class BlockHeightRangeAddressAmounts { + + private BlockHeightRange range; + + private List amounts; + + public BlockHeightRangeAddressAmounts() { + } + + public BlockHeightRangeAddressAmounts(BlockHeightRange range, List amounts) { + this.range = range; + this.amounts = amounts; + } + + public BlockHeightRange getRange() { + return range; + } + + public List getAmounts() { + return amounts; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BlockHeightRangeAddressAmounts that = (BlockHeightRangeAddressAmounts) o; + return Objects.equals(range, that.range) && Objects.equals(amounts, that.amounts); + } + + @Override + public int hashCode() { + return Objects.hash(range, amounts); + } + + @Override + public String toString() { + return "BlockHeightRangeAddressAmounts{" + + "range=" + range + + ", amounts=" + amounts + + '}'; + } +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java index a4aacbf5..f4d33b87 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java @@ -7,11 +7,16 @@ import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.Controller; import org.qortal.data.account.AccountBalanceData; +import org.qortal.data.account.AddressAmountData; +import org.qortal.data.account.BlockHeightRange; +import org.qortal.data.account.BlockHeightRangeAddressAmounts; import org.qortal.data.arbitrary.ArbitraryResourceCache; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.arbitrary.ArbitraryResourceStatus; import org.qortal.repository.DataException; +import org.qortal.settings.Settings; +import org.qortal.utils.BalanceRecorderUtils; import java.sql.ResultSet; import java.sql.SQLException; @@ -28,6 +33,7 @@ import java.util.Optional; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -389,14 +395,15 @@ public class HSQLDBCacheUtils { /** * Start Recording Balances * - * @param queue the queue to add to, remove oldest data if necssary - * @param repository the db repsoitory + * @param balancesByHeight height -> account balances + * @param balanceDynamics every balance dynamic * @param priorityRequested the requested thread priority - * @param frequency the recording frequencies, in minutes + * @param frequency the recording frequencies, in minutes + * @param capacity the maximum size of balanceDynamics */ public static void startRecordingBalances( final ConcurrentHashMap> balancesByHeight, - final ConcurrentHashMap> balancesByAddress, + CopyOnWriteArrayList balanceDynamics, int priorityRequested, int frequency, int capacity) { @@ -409,55 +416,64 @@ public class HSQLDBCacheUtils { Thread.currentThread().setName(BALANCE_RECORDER_TIMER_TASK); - try (final HSQLDBRepository repository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) { - while (balancesByHeight.size() > capacity + 1) { - Optional firstHeight = balancesByHeight.keySet().stream().sorted().findFirst(); + int currentHeight = recordCurrentBalances(balancesByHeight); - if (firstHeight.isPresent()) balancesByHeight.remove(firstHeight.get()); - } + LOGGER.debug("recorded balances: height = " + currentHeight); - // get current balances - List accountBalances = getAccountBalances(repository); + // remove invalidated recordings, recording after current height + BalanceRecorderUtils.removeRecordingsAboveHeight(currentHeight, balancesByHeight); - // get anyone of the balances - Optional data = accountBalances.stream().findAny(); + // remove invalidated dynamics, on or after current height + BalanceRecorderUtils.removeDynamicsOnOrAboveHeight(currentHeight, balanceDynamics); - // if there are any balances, then record them - if (data.isPresent()) { - // map all new balances to the current height - balancesByHeight.put(data.get().getHeight(), accountBalances); + // if there are 2 or more recordings, then produce balance dynamics for the first 2 recordings + if( balancesByHeight.size() > 1 ) { - // for each new balance, map to address - for (AccountBalanceData accountBalance : accountBalances) { + Optional priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight); - // get recorded balances for this address - List establishedBalances - = balancesByAddress.getOrDefault(accountBalance.getAddress(), new ArrayList<>(0)); + // if there is a prior height + if(priorHeight.isPresent()) { - // start a new list of recordings for this address, add the new balance and add the established - // balances - List balances = new ArrayList<>(establishedBalances.size() + 1); - balances.add(accountBalance); - balances.addAll(establishedBalances); + BlockHeightRange blockHeightRange = new BlockHeightRange(priorHeight.get(), currentHeight); - // reset tha balances for this address - balancesByAddress.put(accountBalance.getAddress(), balances); + LOGGER.debug("building dynamics for block heights: range = " + blockHeightRange); - // TODO: reduce account balances to capacity - } + List currentBalances = balancesByHeight.get(currentHeight); - // reduce height balances to capacity - while( balancesByHeight.size() > capacity ) { - Optional lowestHeight - = balancesByHeight.entrySet().stream() - .min(Comparator.comparingInt(Map.Entry::getKey)) - .map(Map.Entry::getKey); + List currentDynamics + = BalanceRecorderUtils.buildBalanceDynamics( + currentBalances, + balancesByHeight.get(priorHeight.get()), + Settings.getInstance().getMinimumBalanceRecording()); - if (lowestHeight.isPresent()) balancesByHeight.entrySet().remove(lowestHeight); + LOGGER.debug("dynamics built: count = " + currentDynamics.size()); + + if(LOGGER.isDebugEnabled()) + currentDynamics.stream() + .sorted(Comparator.comparingLong(AddressAmountData::getAmount).reversed()) + .limit(Settings.getInstance().getTopBalanceLoggingLimit()) + .forEach(top5Dynamic -> LOGGER.debug("Top Dynamics = " + top5Dynamic)); + + BlockHeightRangeAddressAmounts amounts + = new BlockHeightRangeAddressAmounts( blockHeightRange, currentDynamics ); + + balanceDynamics.add(amounts); + + BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight - Settings.getInstance().getBalanceRecorderRollbackAllowance(), balancesByHeight); + + while(balanceDynamics.size() > capacity) { + BlockHeightRangeAddressAmounts oldestDynamics = BalanceRecorderUtils.removeOldestDynamics(balanceDynamics); + + LOGGER.debug("removing oldest dynamics: range " + oldestDynamics.getRange()); } } - } catch (DataException e) { - LOGGER.error(e.getMessage(), e); + else { + LOGGER.warn("Expecting prior height and nothing was discovered, current height = " + currentHeight); + } + } + // else this should be the first recording + else { + LOGGER.info("first balance recording completed"); } } }; @@ -466,6 +482,35 @@ public class HSQLDBCacheUtils { timer.scheduleAtFixedRate(task, 300_000, frequency * 60_000); } + private static int recordCurrentBalances(ConcurrentHashMap> balancesByHeight) { + int currentHeight; + + try (final HSQLDBRepository repository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) { + + // get current balances + List accountBalances = getAccountBalances(repository); + + // get anyone of the balances + Optional data = accountBalances.stream().findAny(); + + // if there are any balances, then record them + if (data.isPresent()) { + // map all new balances to the current height + balancesByHeight.put(data.get().getHeight(), accountBalances); + + currentHeight = data.get().getHeight(); + } + else { + currentHeight = Integer.MAX_VALUE; + } + } catch (DataException e) { + LOGGER.error(e.getMessage(), e); + currentHeight = Integer.MAX_VALUE; + } + + return currentHeight; + } + /** * Build Timer * diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 2a749242..5222f22e 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -444,14 +444,56 @@ public class Settings { */ private long archivingPause = 3000; + /** + * Enable Balance Recorder? + * + * True for balance recording, otherwise false. + */ private boolean balanceRecorderEnabled = false; + /** + * Balance Recorder Priority + * + * The thread priority (1 is lowest, 10 is highest) of the balance recorder thread, if enabled. + */ private int balanceRecorderPriority = 1; - private int balanceRecorderFrequency = 2*60*1000; + /** + * Balance Recorder Frequency + * + * How often the balances will be recorded, if enabled, measured in minutes. + */ + private int balanceRecorderFrequency = 20; + /** + * Balance Recorder Capacity + * + * The number of balance recorder ranges will be held in memory. + */ private int balanceRecorderCapacity = 1000; + /** + * Minimum Balance Recording + * + * The minimum recored balance change in Qortoshis (1/100000000 QORT) + */ + private long minimumBalanceRecording = 100000000; + + /** + * Top Balance Logging Limit + * + * When logging the number limit of top balance changes to show in the logs for any given block range. + */ + private long topBalanceLoggingLimit = 100; + + /** + * Balance Recorder Rollback Allowance + * + * If the balance recorder is enabled, it must protect its prior balances by this number of blocks in case of + * a blockchain rollback and reorganization. + */ + private int balanceRecorderRollbackAllowance = 100; + // Domain mapping public static class ThreadLimit { private String messageType; @@ -1257,4 +1299,16 @@ public class Settings { public boolean isBalanceRecorderEnabled() { return balanceRecorderEnabled; } + + public long getMinimumBalanceRecording() { + return minimumBalanceRecording; + } + + public long getTopBalanceLoggingLimit() { + return topBalanceLoggingLimit; + } + + public int getBalanceRecorderRollbackAllowance() { + return balanceRecorderRollbackAllowance; + } } diff --git a/src/main/java/org/qortal/utils/BalanceRecorderUtils.java b/src/main/java/org/qortal/utils/BalanceRecorderUtils.java new file mode 100644 index 00000000..44d8f5e8 --- /dev/null +++ b/src/main/java/org/qortal/utils/BalanceRecorderUtils.java @@ -0,0 +1,119 @@ +package org.qortal.utils; + +import org.qortal.data.account.AccountBalanceData; +import org.qortal.data.account.AddressAmountData; +import org.qortal.data.account.BlockHeightRange; +import org.qortal.data.account.BlockHeightRangeAddressAmounts; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public class BalanceRecorderUtils { + + public static final Predicate ADDRESS_AMOUNT_DATA_NOT_ZERO = addressAmount -> addressAmount.getAmount() != 0; + public static final Comparator BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR = new Comparator() { + @Override + public int compare(BlockHeightRangeAddressAmounts amounts1, BlockHeightRangeAddressAmounts amounts2) { + return amounts1.getRange().getEnd() - amounts2.getRange().getEnd(); + } + }; + + public static final Comparator ADDRESS_AMOUNT_DATA_COMPARATOR = new Comparator() { + @Override + public int compare(AddressAmountData addressAmountData, AddressAmountData t1) { + if( addressAmountData.getAmount() > t1.getAmount() ) { + return 1; + } + else if( addressAmountData.getAmount() < t1.getAmount() ) { + return -1; + } + else { + return 0; + } + } + }; + + public static final Comparator BLOCK_HEIGHT_RANGE_COMPARATOR = new Comparator() { + @Override + public int compare(BlockHeightRange range1, BlockHeightRange range2) { + return range1.getEnd() - range2.getEnd(); + } + }; + + /** + * Build Balance Dynmaics For Account + * + * @param priorBalances the balances prior to the current height, assuming only one balance per address + * @param accountBalance the current balance + * + * @return the difference between the current balance and the prior balance for the current balance address + */ + public static AddressAmountData buildBalanceDynamicsForAccount(List priorBalances, AccountBalanceData accountBalance) { + Optional matchingAccountPriorBalance + = priorBalances.stream() + .filter(priorBalance -> accountBalance.getAddress().equals(priorBalance.getAddress())) + .findFirst(); + if(matchingAccountPriorBalance.isPresent()) { + return new AddressAmountData(accountBalance.getAddress(), accountBalance.getBalance() - matchingAccountPriorBalance.get().getBalance()); + } + else { + return new AddressAmountData(accountBalance.getAddress(), accountBalance.getBalance()); + } + } + + public static List buildBalanceDynamics(final List balances, final List priorBalances, long minimum) { + + List addressAmounts = new ArrayList<>(balances.size()); + + // prior balance + addressAmounts.addAll( + balances.stream() + .map(balance -> buildBalanceDynamicsForAccount(priorBalances, balance)) + .filter(ADDRESS_AMOUNT_DATA_NOT_ZERO) + .filter( data -> data.getAmount() >= minimum) + .collect(Collectors.toList()) + ); + + return addressAmounts; + } + + public static void removeRecordingsAboveHeight(int currentHeight, ConcurrentHashMap> balancesByHeight) { + balancesByHeight.entrySet().stream() + .filter(heightWithBalances -> heightWithBalances.getKey() > currentHeight) + .forEach(heightWithBalances -> balancesByHeight.remove(heightWithBalances.getKey())); + } + + public static void removeRecordingsBelowHeight(int currentHeight, ConcurrentHashMap> balancesByHeight) { + balancesByHeight.entrySet().stream() + .filter(heightWithBalances -> heightWithBalances.getKey() < currentHeight) + .forEach(heightWithBalances -> balancesByHeight.remove(heightWithBalances.getKey())); + } + + public static void removeDynamicsOnOrAboveHeight(int currentHeight, CopyOnWriteArrayList balanceDynamics) { + balanceDynamics.stream() + .filter(addressAmounts -> addressAmounts.getRange().getEnd() >= currentHeight) + .forEach(addressAmounts -> balanceDynamics.remove(addressAmounts)); + } + + public static BlockHeightRangeAddressAmounts removeOldestDynamics(CopyOnWriteArrayList balanceDynamics) { + BlockHeightRangeAddressAmounts oldestDynamics + = balanceDynamics.stream().sorted(BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR).findFirst().get(); + + balanceDynamics.remove(oldestDynamics); + return oldestDynamics; + } + + public static Optional getPriorHeight(int currentHeight, ConcurrentHashMap> balancesByHeight) { + Optional priorHeight + = balancesByHeight.keySet().stream() + .filter(height -> height < currentHeight) + .sorted(Comparator.reverseOrder()).findFirst(); + return priorHeight; + } +} diff --git a/src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java b/src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java new file mode 100644 index 00000000..746c845f --- /dev/null +++ b/src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java @@ -0,0 +1,361 @@ +package org.qortal.test.utils; + +import org.junit.Assert; +import org.junit.Test; +import org.qortal.data.account.AccountBalanceData; +import org.qortal.data.account.AddressAmountData; +import org.qortal.data.account.BlockHeightRange; +import org.qortal.data.account.BlockHeightRangeAddressAmounts; +import org.qortal.utils.BalanceRecorderUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; + +public class BalanceRecorderUtilsTests { + + @Test + public void testNotZeroForZero() { + boolean test = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_NOT_ZERO.test( new AddressAmountData("", 0)); + + Assert.assertFalse(test); + } + + @Test + public void testNotZeroForPositive() { + boolean test = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_NOT_ZERO.test(new AddressAmountData("", 1)); + + Assert.assertTrue(test); + } + + @Test + public void testNotZeroForNegative() { + boolean test = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_NOT_ZERO.test( new AddressAmountData("", -10)); + + Assert.assertTrue(test); + } + + @Test + public void testAddressAmountComparatorReverseOrder() { + + BlockHeightRangeAddressAmounts addressAmounts1 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(2, 3), new ArrayList<>(0)); + BlockHeightRangeAddressAmounts addressAmounts2 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 2), new ArrayList<>(0)); + + int compare = BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.compare(addressAmounts1, addressAmounts2); + + Assert.assertTrue( compare > 0); + } + + @Test + public void testAddressAmountComparatorForwardOrder() { + + BlockHeightRangeAddressAmounts addressAmounts1 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 2), new ArrayList<>(0)); + BlockHeightRangeAddressAmounts addressAmounts2 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(2, 3), new ArrayList<>(0)); + + int compare = BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.compare(addressAmounts1, addressAmounts2); + + Assert.assertTrue( compare < 0 ); + } + + @Test + public void testAddressAmountDataComparator() { + + AddressAmountData addressAmount1 = new AddressAmountData("a", 10); + AddressAmountData addressAmount2 = new AddressAmountData("b", 20); + + int compare = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_COMPARATOR.compare(addressAmount1, addressAmount2); + + Assert.assertTrue( compare < 0); + } + + @Test + public void testRemoveRecordingsBelowHeightNoBalances() { + + int currentHeight = 5; + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(); + + BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight, balancesByHeight); + + Assert.assertEquals(0, balancesByHeight.size()); + } + + @Test + public void testRemoveRecordingsBelowHeightOneBalanceBelow() { + int currentHeight = 5; + + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(1); + + balancesByHeight.put(1, new ArrayList<>(0)); + + Assert.assertEquals(1, balancesByHeight.size()); + + BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight, balancesByHeight); + + Assert.assertEquals(0, balancesByHeight.size()); + } + + @Test + public void testRemoveRecordingsBelowHeightOneBalanceAbove() { + int currentHeight = 5; + + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(1); + + balancesByHeight.put(10, new ArrayList<>(0)); + + Assert.assertEquals(1, balancesByHeight.size()); + + BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight, balancesByHeight); + + Assert.assertEquals(1, balancesByHeight.size()); + } + + @Test + public void testBuildBalanceDynamicsOneAccountOneChange() { + + String address = "a"; + + List balances = new ArrayList<>(1); + balances.add(new AccountBalanceData(address, 0, 2)); + + List priorBalances = new ArrayList<>(1); + priorBalances.add(new AccountBalanceData(address, 0, 1)); + + List dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, 0); + + Assert.assertNotNull(dynamics); + Assert.assertEquals(1, dynamics.size()); + + AddressAmountData addressAmountData = dynamics.get(0); + Assert.assertNotNull(addressAmountData); + Assert.assertEquals(address, addressAmountData.getAddress()); + Assert.assertEquals(1, addressAmountData.getAmount()); + } + + @Test + public void testBuildBalanceDynamicsOneAccountNoPrior() { + + String address = "a"; + + List balances = new ArrayList<>(1); + balances.add(new AccountBalanceData(address, 0, 2)); + + List priorBalances = new ArrayList<>(0); + + List dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, 0); + + Assert.assertNotNull(dynamics); + Assert.assertEquals(1, dynamics.size()); + + AddressAmountData addressAmountData = dynamics.get(0); + Assert.assertNotNull(addressAmountData); + Assert.assertEquals(address, addressAmountData.getAddress()); + Assert.assertEquals(2, addressAmountData.getAmount()); + } + + @Test + public void testBuildBalanceDynamicsTwoAccountsNegativeValues() { + + String address1 = "a"; + String address2 = "b"; + + List balances = new ArrayList<>(2); + balances.add(new AccountBalanceData(address1, 0, 10_000)); + balances.add(new AccountBalanceData(address2, 0, 100)); + + List priorBalances = new ArrayList<>(2); + priorBalances.add(new AccountBalanceData(address2, 0, 200)); + priorBalances.add(new AccountBalanceData(address1, 0, 5000)); + + List dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, -100L); + + Assert.assertNotNull(dynamics); + Assert.assertEquals(2, dynamics.size()); + + Map amountByAddress + = dynamics.stream() + .collect(Collectors.toMap(dynamic -> dynamic.getAddress(), dynamic -> dynamic.getAmount())); + + Assert.assertTrue(amountByAddress.containsKey(address1)); + + long amount1 = amountByAddress.get(address1); + + Assert.assertNotNull(amount1); + Assert.assertEquals(5000L, amount1 ); + + Assert.assertTrue(amountByAddress.containsKey(address2)); + + long amount2 = amountByAddress.get(address2); + + Assert.assertNotNull(amount2); + Assert.assertEquals(-100L, amount2); + } + + @Test + public void testBuildBalanceDynamicsForAccountNoPriorAnyAccount() { + List priorBalances = new ArrayList<>(0); + AccountBalanceData accountBalance = new AccountBalanceData("a", 0, 10); + + AddressAmountData dynamic = BalanceRecorderUtils.buildBalanceDynamicsForAccount(priorBalances, accountBalance); + + Assert.assertNotNull(dynamic); + Assert.assertEquals(10, dynamic.getAmount()); + Assert.assertEquals("a", dynamic.getAddress()); + } + + @Test + public void testBuildBalanceDynamicsForAccountNoPriorThisAccount() { + List priorBalances = new ArrayList<>(2); + priorBalances.add(new AccountBalanceData("b", 0, 100)); + + AccountBalanceData accountBalanceData = new AccountBalanceData("a", 0, 10); + + AddressAmountData dynamic = BalanceRecorderUtils.buildBalanceDynamicsForAccount(priorBalances, accountBalanceData); + + Assert.assertNotNull(dynamic); + Assert.assertEquals(10, dynamic.getAmount()); + Assert.assertEquals("a", dynamic.getAddress()); + } + + @Test + public void testBuildBalanceDynamicsForAccountPriorForThisAndOthers() { + List priorBalances = new ArrayList<>(2); + priorBalances.add(new AccountBalanceData("a", 0, 100)); + priorBalances.add(new AccountBalanceData("b", 0, 200)); + priorBalances.add(new AccountBalanceData("c", 0, 300)); + + AccountBalanceData accountBalance = new AccountBalanceData("b", 0, 1000); + + AddressAmountData dynamic = BalanceRecorderUtils.buildBalanceDynamicsForAccount(priorBalances, accountBalance); + + Assert.assertNotNull(dynamic); + Assert.assertEquals(800, dynamic.getAmount()); + Assert.assertEquals("b", dynamic.getAddress()); + } + + @Test + public void testRemoveRecordingAboveHeightOneOfTwo() { + + int currentHeight = 10; + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(); + + balancesByHeight.put(3, new ArrayList<>()); + balancesByHeight.put(20, new ArrayList<>()); + + Assert.assertEquals(2, balancesByHeight.size()); + + BalanceRecorderUtils.removeRecordingsAboveHeight(currentHeight, balancesByHeight); + + Assert.assertEquals(1, balancesByHeight.size()); + Assert.assertTrue( balancesByHeight.containsKey(3)); + } + + @Test + public void testPriorHeightBeforeAfter() { + + int currentHeight = 10; + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(); + balancesByHeight.put( 2, new ArrayList<>()); + balancesByHeight.put(7, new ArrayList<>()); + balancesByHeight.put(12, new ArrayList<>()); + + Optional priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight); + + Assert.assertNotNull(priorHeight); + Assert.assertTrue(priorHeight.isPresent()); + Assert.assertEquals( 7, priorHeight.get().intValue()); + } + + @Test + public void testPriorHeightNoPriorAfterOnly() { + + int currentHeight = 10; + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(); + balancesByHeight.put(12, new ArrayList<>()); + + Optional priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight); + + Assert.assertNotNull(priorHeight); + Assert.assertTrue(priorHeight.isEmpty()); + } + + @Test + public void testPriorHeightPriorOnly() { + + int currentHeight = 10; + + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(); + balancesByHeight.put(7, new ArrayList<>()); + + Optional priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight); + + Assert.assertNotNull(priorHeight); + Assert.assertTrue(priorHeight.isPresent()); + Assert.assertEquals(7, priorHeight.get().intValue()); + } + + @Test + public void testRemoveDynamicsOnOrAboveHeightOneAbove() { + + int currentHeight = 10; + + CopyOnWriteArrayList dynamics = new CopyOnWriteArrayList<>(); + + BlockHeightRange range1 = new BlockHeightRange(10, 20); + dynamics.add(new BlockHeightRangeAddressAmounts(range1, new ArrayList<>())); + + BlockHeightRange range2 = new BlockHeightRange(1, 4); + dynamics.add(new BlockHeightRangeAddressAmounts(range2, new ArrayList<>())); + + Assert.assertEquals(2, dynamics.size()); + BalanceRecorderUtils.removeDynamicsOnOrAboveHeight(currentHeight, dynamics); + + Assert.assertEquals(1, dynamics.size()); + Assert.assertEquals(range2, dynamics.get(0).getRange()); + } + + @Test + public void testRemoveDynamicsOnOrAboveOneOnOneAbove() { + int currentHeight = 11; + + CopyOnWriteArrayList dynamics = new CopyOnWriteArrayList<>(); + + BlockHeightRange range1 = new BlockHeightRange(1,5); + dynamics.add(new BlockHeightRangeAddressAmounts(range1, new ArrayList<>())); + + BlockHeightRange range2 = new BlockHeightRange(6, 11); + dynamics.add((new BlockHeightRangeAddressAmounts(range2, new ArrayList<>()))); + + BlockHeightRange range3 = new BlockHeightRange(22, 16); + dynamics.add(new BlockHeightRangeAddressAmounts(range3, new ArrayList<>())); + + Assert.assertEquals(3, dynamics.size()); + + BalanceRecorderUtils.removeDynamicsOnOrAboveHeight(currentHeight, dynamics); + + Assert.assertEquals(1, dynamics.size()); + Assert.assertTrue( dynamics.get(0).getRange().equals(range1)); + } + + @Test + public void testRemoveOldestDynamicsTwice() { + CopyOnWriteArrayList dynamics = new CopyOnWriteArrayList<>(); + + dynamics.add(new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 5), new ArrayList<>())); + dynamics.add(new BlockHeightRangeAddressAmounts(new BlockHeightRange(5, 9), new ArrayList<>())); + + Assert.assertEquals(2, dynamics.size()); + + BalanceRecorderUtils.removeOldestDynamics(dynamics); + + Assert.assertEquals(1, dynamics.size()); + Assert.assertTrue(dynamics.get(0).getRange().equals(new BlockHeightRange(5, 9))); + + BalanceRecorderUtils.removeOldestDynamics(dynamics); + + Assert.assertEquals(0, dynamics.size()); + } +} \ No newline at end of file