Balance Recorder initial implementation.

src/main/java/org/qortal/api/resource/AssetsResource.java

@@ -16,9 +16,13 @@ import org.qortal.api.model.AggregatedOrder;
 import org.qortal.api.model.TradeWithOrderInfo;
 import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
 import org.qortal.asset.Asset;
+import org.qortal.controller.hsqldb.HSQLDBBalanceRecorder;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.account.AccountBalanceData;
 import org.qortal.data.account.AccountData;
+import org.qortal.data.account.AddressAmountData;
+import org.qortal.data.account.BlockHeightRange;
+import org.qortal.data.account.BlockHeightRangeAddressAmounts;
 import org.qortal.data.asset.AssetData;
 import org.qortal.data.asset.OrderData;
 import org.qortal.data.asset.RecentTradeData;
@@ -33,6 +37,7 @@ import org.qortal.transaction.Transaction;
 import org.qortal.transaction.Transaction.ValidationResult;
 import org.qortal.transform.TransformationException;
 import org.qortal.transform.transaction.*;
+import org.qortal.utils.BalanceRecorderUtils;
 import org.qortal.utils.Base58;
 
 import javax.servlet.http.HttpServletRequest;
@@ -42,6 +47,7 @@ import javax.ws.rs.core.MediaType;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 @Path("/assets")
@@ -179,6 +185,122 @@ public class AssetsResource {
 		}
 	}
 
+	@GET
+	@Path("/balancedynamicranges")
+	@Operation(
+			summary = "Get balance dynamic ranges listed.",
+			description = ".",
+			responses = {
+					@ApiResponse(
+							content = @Content(
+									array = @ArraySchema(
+											schema = @Schema(
+													implementation = BlockHeightRange.class
+											)
+									)
+							)
+					)
+			}
+	)
+	public List<BlockHeightRange> getBalanceDynamicRanges(
+			@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
+			@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
+			@Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) {
+
+		Optional<HSQLDBBalanceRecorder> recorder = HSQLDBBalanceRecorder.getInstance();
+
+		if( recorder.isPresent()) {
+			return recorder.get().getRanges(offset, limit, reverse);
+		}
+		else {
+			return new ArrayList<>(0);
+		}
+	}
+
+	@GET
+	@Path("/balancedynamicrange/{height}")
+	@Operation(
+			summary = "Get balance dynamic range for a given height.",
+			description = ".",
+			responses = {
+					@ApiResponse(
+							content = @Content(
+										schema = @Schema(
+												implementation = BlockHeightRange.class
+										)
+							)
+					)
+			}
+	)
+	@ApiErrors({
+			ApiError.INVALID_CRITERIA, ApiError.INVALID_DATA
+	})
+	public BlockHeightRange getBalanceDynamicRange(@PathParam("height") int height) {
+
+		Optional<HSQLDBBalanceRecorder> recorder = HSQLDBBalanceRecorder.getInstance();
+
+		if( recorder.isPresent()) {
+			Optional<BlockHeightRange> range = recorder.get().getRange(height);
+
+			if( range.isPresent() ) {
+				return range.get();
+			}
+			else {
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+			}
+		}
+		else {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+		}
+	}
+
+	@GET
+	@Path("/balancedynamicamounts/{begin}/{end}")
+	@Operation(
+			summary = "Get balance dynamic ranges address amounts listed.",
+			description = ".",
+			responses = {
+					@ApiResponse(
+							content = @Content(
+									array = @ArraySchema(
+											schema = @Schema(
+													implementation = AddressAmountData.class
+											)
+									)
+							)
+					)
+			}
+	)
+	@ApiErrors({
+			ApiError.INVALID_CRITERIA, ApiError.INVALID_DATA
+	})
+	public List<AddressAmountData> getBalanceDynamicAddressAmounts(
+			@PathParam("begin") int begin,
+			@PathParam("end") int end,
+			@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
+			@Parameter(ref = "limit") @QueryParam("limit") Integer limit) {
+
+		Optional<HSQLDBBalanceRecorder> recorder = HSQLDBBalanceRecorder.getInstance();
+
+		if( recorder.isPresent()) {
+			Optional<BlockHeightRangeAddressAmounts> addressAmounts = recorder.get().getAddressAmounts(new BlockHeightRange(begin, end));
+
+			if( addressAmounts.isPresent() ) {
+				return addressAmounts.get().getAmounts().stream()
+					.sorted(BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_COMPARATOR.reversed())
+					.skip(offset)
+					.limit(limit)
+					.collect(Collectors.toList());
+			}
+			else {
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+			}
+		}
+		else {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+		}
+	}
+
 	@GET
 	@Path("/openorders/{assetid}/{otherassetid}")
 	@Operation(

src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java

@@ -2,15 +2,19 @@ package org.qortal.controller.hsqldb;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.util.PropertySource;
 import org.qortal.data.account.AccountBalanceData;
+import org.qortal.data.account.BlockHeightRange;
+import org.qortal.data.account.BlockHeightRangeAddressAmounts;
 import org.qortal.repository.hsqldb.HSQLDBCacheUtils;
 import org.qortal.settings.Settings;
+import org.qortal.utils.BalanceRecorderUtils;
 
-import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.stream.Collectors;
 
 public class HSQLDBBalanceRecorder extends Thread{
@@ -23,6 +27,8 @@ public class HSQLDBBalanceRecorder extends Thread{
 
     private ConcurrentHashMap<String, List<AccountBalanceData>> balancesByAddress = new ConcurrentHashMap<>();
 
+    private CopyOnWriteArrayList<BlockHeightRangeAddressAmounts> balanceDynamics = new CopyOnWriteArrayList<>();
+
     private int priorityRequested;
     private int frequency;
     private int capacity;
@@ -61,36 +67,52 @@ public class HSQLDBBalanceRecorder extends Thread{
 
         Thread.currentThread().setName("Balance Recorder");
 
-        HSQLDBCacheUtils.startRecordingBalances(this.balancesByHeight, this.balancesByAddress, this.priorityRequested, this.frequency, this.capacity);
+        HSQLDBCacheUtils.startRecordingBalances(this.balancesByHeight, this.balanceDynamics, this.priorityRequested, this.frequency, this.capacity);
     }
 
-    public List<AccountBalanceData> getLatestRecordings(int limit, long offset) {
+    public List<BlockHeightRangeAddressAmounts> getLatestDynamics(int limit, long offset) {
-        ArrayList<AccountBalanceData> data;
 
-        Optional<Integer> lastHeight = getLastHeight();
+        List<BlockHeightRangeAddressAmounts> latest = this.balanceDynamics.stream()
+                .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.reversed())
+                .skip(offset)
+                .limit(limit)
+                .collect(Collectors.toList());
 
-        if(lastHeight.isPresent() ) {
+        return latest;
-            List<AccountBalanceData> latest = this.balancesByHeight.get(lastHeight.get());
+    }
 
-            if( latest != null ) {
+    public List<BlockHeightRange> getRanges(Integer offset, Integer limit, Boolean reverse) {
-                data = new ArrayList<>(latest.size());
+
-                data.addAll(
+        if( reverse ) {
-                    latest.stream()
+            return this.balanceDynamics.stream()
-                        .sorted(Comparator.comparingDouble(AccountBalanceData::getBalance).reversed())
+                    .map(BlockHeightRangeAddressAmounts::getRange)
-                        .skip(offset)
+                    .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_COMPARATOR.reversed())
-                        .limit(limit)
+                    .skip(offset)
-                        .collect(Collectors.toList())
+                    .limit(limit)
-                );
+                    .collect(Collectors.toList());
-            }
-            else {
-                data = new ArrayList<>(0);
-            }
         }
         else {
-            data = new ArrayList<>(0);
+            return this.balanceDynamics.stream()
+                    .map(BlockHeightRangeAddressAmounts::getRange)
+                    .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_COMPARATOR)
+                    .skip(offset)
+                    .limit(limit)
+                    .collect(Collectors.toList());
         }
+    }
 
-        return data;
+    public Optional<BlockHeightRangeAddressAmounts> getAddressAmounts(BlockHeightRange range) {
+
+        return this.balanceDynamics.stream()
+            .filter( dynamic -> dynamic.getRange().equals(range))
+            .findAny();
+    }
+
+    public Optional<BlockHeightRange> getRange( int height ) {
+        return this.balanceDynamics.stream()
+            .map(BlockHeightRangeAddressAmounts::getRange)
+            .filter( range -> range.getBegin() < height && range.getEnd() >= height )
+            .findAny();
     }
 
     private Optional<Integer> getLastHeight() {

src/main/java/org/qortal/data/account/AddressAmountData.java

@@ -0,0 +1,54 @@
+package org.qortal.data.account;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+import java.util.Objects;
+
+// All properties to be converted to JSON via JAXB
+@XmlAccessorType(XmlAccessType.FIELD)
+public class AddressAmountData {
+
+    private String address;
+
+    @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
+    private long amount;
+
+    public AddressAmountData() {
+    }
+
+    public AddressAmountData(String address, long amount) {
+
+        this.address = address;
+        this.amount = amount;
+    }
+
+    public String getAddress() {
+        return address;
+    }
+
+    public long getAmount() {
+        return amount;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AddressAmountData that = (AddressAmountData) o;
+        return amount == that.amount && Objects.equals(address, that.address);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(address, amount);
+    }
+
+    @Override
+    public String toString() {
+        return "AddressAmountData{" +
+                "address='" + address + '\'' +
+                ", amount=" + amount +
+                '}';
+    }
+}

src/main/java/org/qortal/data/account/BlockHeightRange.java

@@ -0,0 +1,51 @@
+package org.qortal.data.account;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.Objects;
+
+// All properties to be converted to JSON via JAXB
+@XmlAccessorType(XmlAccessType.FIELD)
+public class BlockHeightRange {
+
+    private int begin;
+
+    private int end;
+
+    public BlockHeightRange() {
+    }
+
+    public BlockHeightRange(int begin, int end) {
+        this.begin = begin;
+        this.end = end;
+    }
+
+    public int getBegin() {
+        return begin;
+    }
+
+    public int getEnd() {
+        return end;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        BlockHeightRange that = (BlockHeightRange) o;
+        return begin == that.begin && end == that.end;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(begin, end);
+    }
+
+    @Override
+    public String toString() {
+        return "BlockHeightRange{" +
+                "begin=" + begin +
+                ", end=" + end +
+                '}';
+    }
+}

src/main/java/org/qortal/data/account/BlockHeightRangeAddressAmounts.java

@@ -0,0 +1,52 @@
+package org.qortal.data.account;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.List;
+import java.util.Objects;
+
+// All properties to be converted to JSON via JAXB
+@XmlAccessorType(XmlAccessType.FIELD)
+public class BlockHeightRangeAddressAmounts {
+
+    private BlockHeightRange range;
+
+    private List<AddressAmountData> amounts;
+
+    public BlockHeightRangeAddressAmounts() {
+    }
+
+    public BlockHeightRangeAddressAmounts(BlockHeightRange range, List<AddressAmountData> amounts) {
+        this.range = range;
+        this.amounts = amounts;
+    }
+
+    public BlockHeightRange getRange() {
+        return range;
+    }
+
+    public List<AddressAmountData> getAmounts() {
+        return amounts;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        BlockHeightRangeAddressAmounts that = (BlockHeightRangeAddressAmounts) o;
+        return Objects.equals(range, that.range) && Objects.equals(amounts, that.amounts);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(range, amounts);
+    }
+
+    @Override
+    public String toString() {
+        return "BlockHeightRangeAddressAmounts{" +
+                "range=" + range +
+                ", amounts=" + amounts +
+                '}';
+    }
+}

src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java

@@ -7,11 +7,16 @@ import org.qortal.arbitrary.misc.Category;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.Controller;
 import org.qortal.data.account.AccountBalanceData;
+import org.qortal.data.account.AddressAmountData;
+import org.qortal.data.account.BlockHeightRange;
+import org.qortal.data.account.BlockHeightRangeAddressAmounts;
 import org.qortal.data.arbitrary.ArbitraryResourceCache;
 import org.qortal.data.arbitrary.ArbitraryResourceData;
 import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
 import org.qortal.data.arbitrary.ArbitraryResourceStatus;
 import org.qortal.repository.DataException;
+import org.qortal.settings.Settings;
+import org.qortal.utils.BalanceRecorderUtils;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -28,6 +33,7 @@ import java.util.Optional;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
@@ -389,14 +395,15 @@ public class HSQLDBCacheUtils {
     /**
      * Start Recording Balances
      *
-     * @param queue             the queue to add to, remove oldest data if necssary
+     * @param balancesByHeight height -> account balances
-     * @param repository        the db repsoitory
+     * @param balanceDynamics every balance dynamic
      * @param priorityRequested the requested thread priority
-     * @param frequency         the recording frequencies, in minutes
+     * @param frequency the recording frequencies, in minutes
+     * @param capacity the maximum size of balanceDynamics
      */
     public static void startRecordingBalances(
             final ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight,
-            final ConcurrentHashMap<String, List<AccountBalanceData>> balancesByAddress,
+            CopyOnWriteArrayList<BlockHeightRangeAddressAmounts> balanceDynamics,
             int priorityRequested,
             int frequency,
             int capacity) {
@@ -409,55 +416,64 @@ public class HSQLDBCacheUtils {
 
                 Thread.currentThread().setName(BALANCE_RECORDER_TIMER_TASK);
 
-                try (final HSQLDBRepository repository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) {
+                int currentHeight = recordCurrentBalances(balancesByHeight);
-                    while (balancesByHeight.size() > capacity + 1) {
-                        Optional<Integer> firstHeight = balancesByHeight.keySet().stream().sorted().findFirst();
 
-                        if (firstHeight.isPresent()) balancesByHeight.remove(firstHeight.get());
+                LOGGER.debug("recorded balances: height = " + currentHeight);
-                    }
 
-                    // get current balances
+                // remove invalidated recordings, recording after current height
-                    List<AccountBalanceData> accountBalances = getAccountBalances(repository);
+                BalanceRecorderUtils.removeRecordingsAboveHeight(currentHeight, balancesByHeight);
 
-                    // get anyone of the balances
+                // remove invalidated dynamics, on or after current height
-                    Optional<AccountBalanceData> data = accountBalances.stream().findAny();
+                BalanceRecorderUtils.removeDynamicsOnOrAboveHeight(currentHeight, balanceDynamics);
 
-                    // if there are any balances, then record them
+                // if there are 2 or more recordings, then produce balance dynamics for the first 2 recordings
-                    if (data.isPresent()) {
+                if( balancesByHeight.size() > 1 ) {
-                        // map all new balances to the current height
-                        balancesByHeight.put(data.get().getHeight(), accountBalances);
 
-                        // for each new balance, map to address
+                    Optional<Integer> priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight);
-                        for (AccountBalanceData accountBalance : accountBalances) {
 
-                            // get recorded balances for this address
+                    // if there is a prior height
-                            List<AccountBalanceData> establishedBalances
+                    if(priorHeight.isPresent()) {
-                                    = balancesByAddress.getOrDefault(accountBalance.getAddress(), new ArrayList<>(0));
 
-                            // start a new list of recordings for this address, add the new balance and add the established
+                        BlockHeightRange blockHeightRange = new BlockHeightRange(priorHeight.get(), currentHeight);
-                            // balances
-                            List<AccountBalanceData> balances = new ArrayList<>(establishedBalances.size() + 1);
-                            balances.add(accountBalance);
-                            balances.addAll(establishedBalances);
 
-                            // reset tha balances for this address
+                        LOGGER.debug("building dynamics for block heights: range = " + blockHeightRange);
-                            balancesByAddress.put(accountBalance.getAddress(), balances);
 
-                            // TODO: reduce account balances to capacity
+                        List<AccountBalanceData> currentBalances = balancesByHeight.get(currentHeight);
-                        }
 
-                        // reduce height balances to capacity
+                        List<AddressAmountData> currentDynamics
-                        while( balancesByHeight.size() > capacity ) {
+                            = BalanceRecorderUtils.buildBalanceDynamics(
-                            Optional<Integer> lowestHeight
+                                currentBalances,
-                                = balancesByHeight.entrySet().stream()
+                                balancesByHeight.get(priorHeight.get()),
-                                    .min(Comparator.comparingInt(Map.Entry::getKey))
+                                Settings.getInstance().getMinimumBalanceRecording());
-                                    .map(Map.Entry::getKey);
 
-                            if (lowestHeight.isPresent()) balancesByHeight.entrySet().remove(lowestHeight);
+                        LOGGER.debug("dynamics built: count = " + currentDynamics.size());
+
+                        if(LOGGER.isDebugEnabled())
+                            currentDynamics.stream()
+                                .sorted(Comparator.comparingLong(AddressAmountData::getAmount).reversed())
+                                .limit(Settings.getInstance().getTopBalanceLoggingLimit())
+                                .forEach(top5Dynamic -> LOGGER.debug("Top Dynamics = " + top5Dynamic));
+
+                        BlockHeightRangeAddressAmounts amounts
+                            = new BlockHeightRangeAddressAmounts( blockHeightRange, currentDynamics );
+
+                        balanceDynamics.add(amounts);
+
+                        BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight - Settings.getInstance().getBalanceRecorderRollbackAllowance(), balancesByHeight);
+
+                        while(balanceDynamics.size() > capacity) {
+                            BlockHeightRangeAddressAmounts oldestDynamics = BalanceRecorderUtils.removeOldestDynamics(balanceDynamics);
+
+                            LOGGER.debug("removing oldest dynamics: range " + oldestDynamics.getRange());
                         }
                     }
-                } catch (DataException e) {
+                    else {
-                    LOGGER.error(e.getMessage(), e);
+                        LOGGER.warn("Expecting prior height and nothing was discovered, current height = " + currentHeight);
+                    }
+                }
+                // else this should be the first recording
+                else {
+                    LOGGER.info("first balance recording completed");
                 }
             }
         };
@@ -466,6 +482,35 @@ public class HSQLDBCacheUtils {
         timer.scheduleAtFixedRate(task, 300_000, frequency * 60_000);
     }
 
+    private static int recordCurrentBalances(ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight) {
+        int currentHeight;
+
+        try (final HSQLDBRepository repository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) {
+
+            // get current balances
+            List<AccountBalanceData> accountBalances = getAccountBalances(repository);
+
+            // get anyone of the balances
+            Optional<AccountBalanceData> data = accountBalances.stream().findAny();
+
+            // if there are any balances, then record them
+            if (data.isPresent()) {
+                // map all new balances to the current height
+                balancesByHeight.put(data.get().getHeight(), accountBalances);
+
+                currentHeight =  data.get().getHeight();
+            }
+            else {
+                currentHeight = Integer.MAX_VALUE;
+            }
+        } catch (DataException e) {
+            LOGGER.error(e.getMessage(), e);
+            currentHeight = Integer.MAX_VALUE;
+        }
+
+        return currentHeight;
+    }
+
     /**
      * Build Timer
      *

src/main/java/org/qortal/settings/Settings.java

@@ -444,14 +444,56 @@ public class Settings {
 	 */
 	private long archivingPause = 3000;
 
+	/**
+	 * Enable Balance Recorder?
+	 *
+	 * True for balance recording, otherwise false.
+	 */
 	private boolean balanceRecorderEnabled = false;
 
+	/**
+	 * Balance Recorder Priority
+	 *
+	 * The thread priority (1 is lowest, 10 is highest) of the balance recorder thread, if enabled.
+	 */
 	private int balanceRecorderPriority = 1;
 
-	private int balanceRecorderFrequency = 2*60*1000;
+	/**
+	 * Balance Recorder Frequency
+	 *
+	 * How often the balances will be recorded, if enabled, measured in minutes.
+	 */
+	private int balanceRecorderFrequency = 20;
 
+	/**
+	 * Balance Recorder Capacity
+	 *
+	 * The number of balance recorder ranges will be held in memory.
+	 */
 	private int balanceRecorderCapacity = 1000;
 
+	/**
+	 * Minimum Balance Recording
+	 *
+	 * The minimum recored balance change in Qortoshis (1/100000000 QORT)
+	 */
+    private long minimumBalanceRecording = 100000000;
+
+	/**
+	 * Top Balance Logging Limit
+	 *
+	 * When logging the number limit of top balance changes to show in the logs for any given block range.
+	 */
+	private long topBalanceLoggingLimit = 100;
+
+	/**
+	 * Balance Recorder Rollback Allowance
+	 *
+	 * If the balance recorder is enabled, it must protect its prior balances by this number of blocks in case of
+	 * a blockchain rollback and reorganization.
+	 */
+	private int balanceRecorderRollbackAllowance = 100;
+
 	// Domain mapping
 	public static class ThreadLimit {
 		private String messageType;
@@ -1257,4 +1299,16 @@ public class Settings {
 	public boolean isBalanceRecorderEnabled() {
 		return balanceRecorderEnabled;
 	}
+
+	public long getMinimumBalanceRecording() {
+		return minimumBalanceRecording;
+	}
+
+	public long getTopBalanceLoggingLimit() {
+		return topBalanceLoggingLimit;
+	}
+
+	public int getBalanceRecorderRollbackAllowance() {
+		return balanceRecorderRollbackAllowance;
+	}
 }

src/main/java/org/qortal/utils/BalanceRecorderUtils.java

@@ -0,0 +1,119 @@
+package org.qortal.utils;
+
+import org.qortal.data.account.AccountBalanceData;
+import org.qortal.data.account.AddressAmountData;
+import org.qortal.data.account.BlockHeightRange;
+import org.qortal.data.account.BlockHeightRangeAddressAmounts;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+public class BalanceRecorderUtils {
+
+    public static final Predicate<AddressAmountData> ADDRESS_AMOUNT_DATA_NOT_ZERO = addressAmount -> addressAmount.getAmount() != 0;
+    public static final Comparator<BlockHeightRangeAddressAmounts> BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR = new Comparator<BlockHeightRangeAddressAmounts>() {
+        @Override
+        public int compare(BlockHeightRangeAddressAmounts amounts1, BlockHeightRangeAddressAmounts amounts2) {
+            return amounts1.getRange().getEnd() - amounts2.getRange().getEnd();
+        }
+    };
+
+    public static final Comparator<AddressAmountData> ADDRESS_AMOUNT_DATA_COMPARATOR = new Comparator<AddressAmountData>() {
+        @Override
+        public int compare(AddressAmountData addressAmountData, AddressAmountData t1) {
+            if( addressAmountData.getAmount() > t1.getAmount() ) {
+                return 1;
+            }
+            else if( addressAmountData.getAmount() < t1.getAmount() ) {
+                return -1;
+            }
+            else {
+                return 0;
+            }
+        }
+    };
+
+    public static final Comparator<BlockHeightRange> BLOCK_HEIGHT_RANGE_COMPARATOR = new Comparator<BlockHeightRange>() {
+        @Override
+        public int compare(BlockHeightRange range1, BlockHeightRange range2) {
+            return range1.getEnd() - range2.getEnd();
+        }
+    };
+
+    /**
+     * Build Balance Dynmaics For Account
+     *
+     * @param priorBalances the balances prior to the current height, assuming only one balance per address
+     * @param accountBalance the current balance
+     *
+     * @return the difference between the current balance and the prior balance for the current balance address
+     */
+    public static AddressAmountData buildBalanceDynamicsForAccount(List<AccountBalanceData> priorBalances, AccountBalanceData accountBalance) {
+        Optional<AccountBalanceData> matchingAccountPriorBalance
+            = priorBalances.stream()
+                .filter(priorBalance -> accountBalance.getAddress().equals(priorBalance.getAddress()))
+                .findFirst();
+        if(matchingAccountPriorBalance.isPresent()) {
+            return new AddressAmountData(accountBalance.getAddress(), accountBalance.getBalance() - matchingAccountPriorBalance.get().getBalance());
+        }
+        else {
+            return new AddressAmountData(accountBalance.getAddress(), accountBalance.getBalance());
+        }
+    }
+
+    public static List<AddressAmountData> buildBalanceDynamics(final List<AccountBalanceData> balances, final List<AccountBalanceData> priorBalances, long minimum) {
+
+        List<AddressAmountData> addressAmounts = new ArrayList<>(balances.size());
+
+        // prior balance
+        addressAmounts.addAll(
+            balances.stream()
+                .map(balance -> buildBalanceDynamicsForAccount(priorBalances, balance))
+                .filter(ADDRESS_AMOUNT_DATA_NOT_ZERO)
+                .filter( data -> data.getAmount() >= minimum)
+                .collect(Collectors.toList())
+        );
+
+        return addressAmounts;
+    }
+
+    public static void removeRecordingsAboveHeight(int currentHeight, ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight) {
+        balancesByHeight.entrySet().stream()
+            .filter(heightWithBalances -> heightWithBalances.getKey() > currentHeight)
+            .forEach(heightWithBalances -> balancesByHeight.remove(heightWithBalances.getKey()));
+    }
+
+    public static void removeRecordingsBelowHeight(int currentHeight, ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight) {
+        balancesByHeight.entrySet().stream()
+            .filter(heightWithBalances -> heightWithBalances.getKey() < currentHeight)
+            .forEach(heightWithBalances -> balancesByHeight.remove(heightWithBalances.getKey()));
+    }
+
+    public static void removeDynamicsOnOrAboveHeight(int currentHeight, CopyOnWriteArrayList<BlockHeightRangeAddressAmounts> balanceDynamics) {
+        balanceDynamics.stream()
+            .filter(addressAmounts -> addressAmounts.getRange().getEnd() >= currentHeight)
+            .forEach(addressAmounts -> balanceDynamics.remove(addressAmounts));
+    }
+
+    public static BlockHeightRangeAddressAmounts removeOldestDynamics(CopyOnWriteArrayList<BlockHeightRangeAddressAmounts> balanceDynamics) {
+        BlockHeightRangeAddressAmounts oldestDynamics
+                = balanceDynamics.stream().sorted(BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR).findFirst().get();
+
+        balanceDynamics.remove(oldestDynamics);
+        return oldestDynamics;
+    }
+
+    public static Optional<Integer> getPriorHeight(int currentHeight, ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight) {
+        Optional<Integer> priorHeight
+            = balancesByHeight.keySet().stream()
+                .filter(height -> height < currentHeight)
+                .sorted(Comparator.reverseOrder()).findFirst();
+        return priorHeight;
+    }
+}

src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java

@@ -0,0 +1,361 @@
+package org.qortal.test.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.qortal.data.account.AccountBalanceData;
+import org.qortal.data.account.AddressAmountData;
+import org.qortal.data.account.BlockHeightRange;
+import org.qortal.data.account.BlockHeightRangeAddressAmounts;
+import org.qortal.utils.BalanceRecorderUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Collectors;
+
+public class BalanceRecorderUtilsTests {
+
+    @Test
+    public void testNotZeroForZero() {
+        boolean test = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_NOT_ZERO.test( new AddressAmountData("", 0));
+
+        Assert.assertFalse(test);
+    }
+
+    @Test
+    public void testNotZeroForPositive() {
+        boolean test = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_NOT_ZERO.test(new AddressAmountData("", 1));
+
+        Assert.assertTrue(test);
+    }
+
+    @Test
+    public void testNotZeroForNegative() {
+        boolean test = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_NOT_ZERO.test( new AddressAmountData("", -10));
+
+        Assert.assertTrue(test);
+    }
+
+    @Test
+    public void testAddressAmountComparatorReverseOrder() {
+
+        BlockHeightRangeAddressAmounts addressAmounts1 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(2, 3), new ArrayList<>(0));
+        BlockHeightRangeAddressAmounts addressAmounts2 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 2), new ArrayList<>(0));
+
+        int compare = BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.compare(addressAmounts1, addressAmounts2);
+
+        Assert.assertTrue( compare > 0);
+    }
+
+    @Test
+    public void testAddressAmountComparatorForwardOrder() {
+
+        BlockHeightRangeAddressAmounts addressAmounts1 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 2), new ArrayList<>(0));
+        BlockHeightRangeAddressAmounts addressAmounts2 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(2, 3), new ArrayList<>(0));
+
+        int compare = BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.compare(addressAmounts1, addressAmounts2);
+
+        Assert.assertTrue( compare < 0 );
+    }
+
+    @Test
+    public void testAddressAmountDataComparator() {
+
+        AddressAmountData addressAmount1 = new AddressAmountData("a", 10);
+        AddressAmountData addressAmount2 = new AddressAmountData("b", 20);
+
+        int compare = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_COMPARATOR.compare(addressAmount1, addressAmount2);
+
+        Assert.assertTrue( compare < 0);
+    }
+
+    @Test
+    public void testRemoveRecordingsBelowHeightNoBalances() {
+
+        int currentHeight = 5;
+        ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight = new ConcurrentHashMap<>();
+
+        BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight, balancesByHeight);
+
+        Assert.assertEquals(0, balancesByHeight.size());
+    }
+
+    @Test
+    public void testRemoveRecordingsBelowHeightOneBalanceBelow() {
+        int currentHeight = 5;
+
+        ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight = new ConcurrentHashMap<>(1);
+
+        balancesByHeight.put(1, new ArrayList<>(0));
+
+        Assert.assertEquals(1, balancesByHeight.size());
+
+        BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight, balancesByHeight);
+
+        Assert.assertEquals(0, balancesByHeight.size());
+    }
+
+    @Test
+    public void testRemoveRecordingsBelowHeightOneBalanceAbove() {
+        int currentHeight = 5;
+
+        ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight = new ConcurrentHashMap<>(1);
+
+        balancesByHeight.put(10, new ArrayList<>(0));
+
+        Assert.assertEquals(1, balancesByHeight.size());
+
+        BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight, balancesByHeight);
+
+        Assert.assertEquals(1, balancesByHeight.size());
+    }
+
+    @Test
+    public void testBuildBalanceDynamicsOneAccountOneChange() {
+
+        String address = "a";
+
+        List<AccountBalanceData> balances = new ArrayList<>(1);
+        balances.add(new AccountBalanceData(address, 0, 2));
+
+        List<AccountBalanceData> priorBalances = new ArrayList<>(1);
+        priorBalances.add(new AccountBalanceData(address, 0, 1));
+
+        List<AddressAmountData> dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, 0);
+
+        Assert.assertNotNull(dynamics);
+        Assert.assertEquals(1, dynamics.size());
+
+        AddressAmountData addressAmountData = dynamics.get(0);
+        Assert.assertNotNull(addressAmountData);
+        Assert.assertEquals(address, addressAmountData.getAddress());
+        Assert.assertEquals(1, addressAmountData.getAmount());
+    }
+
+    @Test
+    public void testBuildBalanceDynamicsOneAccountNoPrior() {
+
+        String address = "a";
+
+        List<AccountBalanceData> balances = new ArrayList<>(1);
+        balances.add(new AccountBalanceData(address, 0, 2));
+
+        List<AccountBalanceData> priorBalances = new ArrayList<>(0);
+
+        List<AddressAmountData> dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, 0);
+
+        Assert.assertNotNull(dynamics);
+        Assert.assertEquals(1, dynamics.size());
+
+        AddressAmountData addressAmountData = dynamics.get(0);
+        Assert.assertNotNull(addressAmountData);
+        Assert.assertEquals(address, addressAmountData.getAddress());
+        Assert.assertEquals(2, addressAmountData.getAmount());
+    }
+
+    @Test
+    public void testBuildBalanceDynamicsTwoAccountsNegativeValues() {
+
+        String address1 = "a";
+        String address2 = "b";
+
+        List<AccountBalanceData> balances = new ArrayList<>(2);
+        balances.add(new AccountBalanceData(address1, 0, 10_000));
+        balances.add(new AccountBalanceData(address2, 0, 100));
+
+        List<AccountBalanceData> priorBalances = new ArrayList<>(2);
+        priorBalances.add(new AccountBalanceData(address2, 0, 200));
+        priorBalances.add(new AccountBalanceData(address1, 0, 5000));
+
+        List<AddressAmountData> dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, -100L);
+
+        Assert.assertNotNull(dynamics);
+        Assert.assertEquals(2, dynamics.size());
+
+        Map<String, Long> amountByAddress
+            = dynamics.stream()
+                .collect(Collectors.toMap(dynamic -> dynamic.getAddress(), dynamic -> dynamic.getAmount()));
+
+        Assert.assertTrue(amountByAddress.containsKey(address1));
+
+        long amount1 = amountByAddress.get(address1);
+
+        Assert.assertNotNull(amount1);
+        Assert.assertEquals(5000L, amount1 );
+
+        Assert.assertTrue(amountByAddress.containsKey(address2));
+
+        long amount2 = amountByAddress.get(address2);
+
+        Assert.assertNotNull(amount2);
+        Assert.assertEquals(-100L, amount2);
+    }
+
+    @Test
+    public void testBuildBalanceDynamicsForAccountNoPriorAnyAccount() {
+        List<AccountBalanceData> priorBalances = new ArrayList<>(0);
+        AccountBalanceData accountBalance = new AccountBalanceData("a", 0, 10);
+
+        AddressAmountData dynamic = BalanceRecorderUtils.buildBalanceDynamicsForAccount(priorBalances, accountBalance);
+
+        Assert.assertNotNull(dynamic);
+        Assert.assertEquals(10, dynamic.getAmount());
+        Assert.assertEquals("a", dynamic.getAddress());
+    }
+
+    @Test
+    public void testBuildBalanceDynamicsForAccountNoPriorThisAccount() {
+        List<AccountBalanceData> priorBalances = new ArrayList<>(2);
+        priorBalances.add(new AccountBalanceData("b", 0, 100));
+
+        AccountBalanceData accountBalanceData = new AccountBalanceData("a", 0, 10);
+
+        AddressAmountData dynamic = BalanceRecorderUtils.buildBalanceDynamicsForAccount(priorBalances, accountBalanceData);
+
+        Assert.assertNotNull(dynamic);
+        Assert.assertEquals(10, dynamic.getAmount());
+        Assert.assertEquals("a", dynamic.getAddress());
+    }
+
+    @Test
+    public void testBuildBalanceDynamicsForAccountPriorForThisAndOthers() {
+        List<AccountBalanceData> priorBalances = new ArrayList<>(2);
+        priorBalances.add(new AccountBalanceData("a", 0, 100));
+        priorBalances.add(new AccountBalanceData("b", 0, 200));
+        priorBalances.add(new AccountBalanceData("c", 0, 300));
+
+        AccountBalanceData accountBalance = new AccountBalanceData("b", 0, 1000);
+
+        AddressAmountData dynamic = BalanceRecorderUtils.buildBalanceDynamicsForAccount(priorBalances, accountBalance);
+
+        Assert.assertNotNull(dynamic);
+        Assert.assertEquals(800, dynamic.getAmount());
+        Assert.assertEquals("b", dynamic.getAddress());
+    }
+
+    @Test
+    public void testRemoveRecordingAboveHeightOneOfTwo() {
+
+        int currentHeight = 10;
+        ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight = new ConcurrentHashMap<>();
+
+        balancesByHeight.put(3, new ArrayList<>());
+        balancesByHeight.put(20, new ArrayList<>());
+
+        Assert.assertEquals(2, balancesByHeight.size());
+
+        BalanceRecorderUtils.removeRecordingsAboveHeight(currentHeight, balancesByHeight);
+
+        Assert.assertEquals(1, balancesByHeight.size());
+        Assert.assertTrue( balancesByHeight.containsKey(3));
+    }
+
+    @Test
+    public void testPriorHeightBeforeAfter() {
+
+        int currentHeight = 10;
+        ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight = new ConcurrentHashMap<>();
+        balancesByHeight.put( 2, new ArrayList<>());
+        balancesByHeight.put(7, new ArrayList<>());
+        balancesByHeight.put(12, new ArrayList<>());
+
+        Optional<Integer> priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight);
+
+        Assert.assertNotNull(priorHeight);
+        Assert.assertTrue(priorHeight.isPresent());
+        Assert.assertEquals( 7, priorHeight.get().intValue());
+    }
+
+    @Test
+    public void testPriorHeightNoPriorAfterOnly() {
+
+        int currentHeight = 10;
+        ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight = new ConcurrentHashMap<>();
+        balancesByHeight.put(12, new ArrayList<>());
+
+        Optional<Integer> priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight);
+
+        Assert.assertNotNull(priorHeight);
+        Assert.assertTrue(priorHeight.isEmpty());
+    }
+
+    @Test
+    public void testPriorHeightPriorOnly() {
+
+        int currentHeight = 10;
+
+        ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight = new ConcurrentHashMap<>();
+        balancesByHeight.put(7, new ArrayList<>());
+
+        Optional<Integer> priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight);
+
+        Assert.assertNotNull(priorHeight);
+        Assert.assertTrue(priorHeight.isPresent());
+        Assert.assertEquals(7, priorHeight.get().intValue());
+    }
+
+    @Test
+    public void testRemoveDynamicsOnOrAboveHeightOneAbove() {
+
+        int currentHeight = 10;
+
+        CopyOnWriteArrayList<BlockHeightRangeAddressAmounts> dynamics = new CopyOnWriteArrayList<>();
+
+        BlockHeightRange range1 = new BlockHeightRange(10, 20);
+        dynamics.add(new BlockHeightRangeAddressAmounts(range1, new ArrayList<>()));
+
+        BlockHeightRange range2 = new BlockHeightRange(1, 4);
+        dynamics.add(new BlockHeightRangeAddressAmounts(range2, new ArrayList<>()));
+
+        Assert.assertEquals(2, dynamics.size());
+        BalanceRecorderUtils.removeDynamicsOnOrAboveHeight(currentHeight, dynamics);
+
+        Assert.assertEquals(1, dynamics.size());
+        Assert.assertEquals(range2, dynamics.get(0).getRange());
+    }
+
+    @Test
+    public void testRemoveDynamicsOnOrAboveOneOnOneAbove() {
+        int currentHeight = 11;
+
+        CopyOnWriteArrayList<BlockHeightRangeAddressAmounts> dynamics = new CopyOnWriteArrayList<>();
+
+        BlockHeightRange range1 = new BlockHeightRange(1,5);
+        dynamics.add(new BlockHeightRangeAddressAmounts(range1, new ArrayList<>()));
+
+        BlockHeightRange range2 = new BlockHeightRange(6, 11);
+        dynamics.add((new BlockHeightRangeAddressAmounts(range2, new ArrayList<>())));
+
+        BlockHeightRange range3 = new BlockHeightRange(22, 16);
+        dynamics.add(new BlockHeightRangeAddressAmounts(range3, new ArrayList<>()));
+
+        Assert.assertEquals(3, dynamics.size());
+
+        BalanceRecorderUtils.removeDynamicsOnOrAboveHeight(currentHeight, dynamics);
+
+        Assert.assertEquals(1, dynamics.size());
+        Assert.assertTrue( dynamics.get(0).getRange().equals(range1));
+    }
+
+    @Test
+    public void testRemoveOldestDynamicsTwice() {
+        CopyOnWriteArrayList<BlockHeightRangeAddressAmounts> dynamics = new CopyOnWriteArrayList<>();
+
+        dynamics.add(new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 5), new ArrayList<>()));
+        dynamics.add(new BlockHeightRangeAddressAmounts(new BlockHeightRange(5, 9), new ArrayList<>()));
+
+        Assert.assertEquals(2, dynamics.size());
+
+        BalanceRecorderUtils.removeOldestDynamics(dynamics);
+
+        Assert.assertEquals(1, dynamics.size());
+        Assert.assertTrue(dynamics.get(0).getRange().equals(new BlockHeightRange(5, 9)));
+
+        BalanceRecorderUtils.removeOldestDynamics(dynamics);
+
+        Assert.assertEquals(0, dynamics.size());
+    }
+}