From 5346c97922e1a170ece07863ec8e7371b512f50c Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 10 Dec 2024 14:07:11 -0800 Subject: [PATCH 01/24] added logging to help solve the updated field problem, the problem is the updated field is not getting updated --- .../org/qortal/transaction/ArbitraryTransaction.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 1a9f888b..e5fa4f58 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -397,11 +397,14 @@ public class ArbitraryTransaction extends Transaction { ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository() .getArbitraryResource(service, name, identifier); + LOGGER.info("updating existing arbitraryResourceData" + existingArbitraryResourceData); + // Check for existing cached data if (existingArbitraryResourceData == null) { // Nothing exists yet, so set creation date from the current transaction (it will be reduced later if needed) arbitraryResourceData.created = arbitraryTransactionData.getTimestamp(); arbitraryResourceData.updated = null; + LOGGER.info("updated = null, reason = existingArbitraryResourceData == null" ); } else { // An entry already exists - update created time from current transaction if this is older @@ -411,14 +414,20 @@ public class ArbitraryTransaction extends Transaction { if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) { // Latest transaction matches created time, so it hasn't been updated arbitraryResourceData.updated = null; + LOGGER.info( + "updated = null, reason: existingArbitraryResourceData.created == latestTransactionData.getTimestamp() == " + + existingArbitraryResourceData.created ); } else { arbitraryResourceData.updated = latestTransactionData.getTimestamp(); + LOGGER.info("setting updated to a non-null value"); } } arbitraryResourceData.size = latestTransactionData.getSize(); + LOGGER.info("saving updated arbitraryResourceData: updated = " + arbitraryResourceData.updated); + // Save repository.getArbitraryRepository().save(arbitraryResourceData); } From 5e145de52bf889916b60ada1bf19177ce03cb3a3 Mon Sep 17 00:00:00 2001 From: kennycud Date: Thu, 12 Dec 2024 13:46:18 -0800 Subject: [PATCH 02/24] Balance Recorder initial implementation. --- .../qortal/api/resource/AssetsResource.java | 122 ++++++ .../hsqldb/HSQLDBBalanceRecorder.java | 66 ++-- .../data/account/AddressAmountData.java | 54 +++ .../qortal/data/account/BlockHeightRange.java | 51 +++ .../BlockHeightRangeAddressAmounts.java | 52 +++ .../repository/hsqldb/HSQLDBCacheUtils.java | 125 ++++-- .../java/org/qortal/settings/Settings.java | 56 ++- .../qortal/utils/BalanceRecorderUtils.java | 119 ++++++ .../test/utils/BalanceRecorderUtilsTests.java | 361 ++++++++++++++++++ 9 files changed, 943 insertions(+), 63 deletions(-) create mode 100644 src/main/java/org/qortal/data/account/AddressAmountData.java create mode 100644 src/main/java/org/qortal/data/account/BlockHeightRange.java create mode 100644 src/main/java/org/qortal/data/account/BlockHeightRangeAddressAmounts.java create mode 100644 src/main/java/org/qortal/utils/BalanceRecorderUtils.java create mode 100644 src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java diff --git a/src/main/java/org/qortal/api/resource/AssetsResource.java b/src/main/java/org/qortal/api/resource/AssetsResource.java index 40e04256..6de01434 100644 --- a/src/main/java/org/qortal/api/resource/AssetsResource.java +++ b/src/main/java/org/qortal/api/resource/AssetsResource.java @@ -16,9 +16,13 @@ import org.qortal.api.model.AggregatedOrder; import org.qortal.api.model.TradeWithOrderInfo; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; import org.qortal.asset.Asset; +import org.qortal.controller.hsqldb.HSQLDBBalanceRecorder; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; +import org.qortal.data.account.AddressAmountData; +import org.qortal.data.account.BlockHeightRange; +import org.qortal.data.account.BlockHeightRangeAddressAmounts; import org.qortal.data.asset.AssetData; import org.qortal.data.asset.OrderData; import org.qortal.data.asset.RecentTradeData; @@ -33,6 +37,7 @@ import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.ValidationResult; import org.qortal.transform.TransformationException; import org.qortal.transform.transaction.*; +import org.qortal.utils.BalanceRecorderUtils; import org.qortal.utils.Base58; import javax.servlet.http.HttpServletRequest; @@ -42,6 +47,7 @@ import javax.ws.rs.core.MediaType; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; @Path("/assets") @@ -179,6 +185,122 @@ public class AssetsResource { } } + @GET + @Path("/balancedynamicranges") + @Operation( + summary = "Get balance dynamic ranges listed.", + description = ".", + responses = { + @ApiResponse( + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = BlockHeightRange.class + ) + ) + ) + ) + } + ) + public List getBalanceDynamicRanges( + @Parameter(ref = "offset") @QueryParam("offset") Integer offset, + @Parameter(ref = "limit") @QueryParam("limit") Integer limit, + @Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) { + + Optional recorder = HSQLDBBalanceRecorder.getInstance(); + + if( recorder.isPresent()) { + return recorder.get().getRanges(offset, limit, reverse); + } + else { + return new ArrayList<>(0); + } + } + + @GET + @Path("/balancedynamicrange/{height}") + @Operation( + summary = "Get balance dynamic range for a given height.", + description = ".", + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + implementation = BlockHeightRange.class + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.INVALID_CRITERIA, ApiError.INVALID_DATA + }) + public BlockHeightRange getBalanceDynamicRange(@PathParam("height") int height) { + + Optional recorder = HSQLDBBalanceRecorder.getInstance(); + + if( recorder.isPresent()) { + Optional range = recorder.get().getRange(height); + + if( range.isPresent() ) { + return range.get(); + } + else { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + } + else { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + } + + @GET + @Path("/balancedynamicamounts/{begin}/{end}") + @Operation( + summary = "Get balance dynamic ranges address amounts listed.", + description = ".", + responses = { + @ApiResponse( + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = AddressAmountData.class + ) + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.INVALID_CRITERIA, ApiError.INVALID_DATA + }) + public List getBalanceDynamicAddressAmounts( + @PathParam("begin") int begin, + @PathParam("end") int end, + @Parameter(ref = "offset") @QueryParam("offset") Integer offset, + @Parameter(ref = "limit") @QueryParam("limit") Integer limit) { + + Optional recorder = HSQLDBBalanceRecorder.getInstance(); + + if( recorder.isPresent()) { + Optional addressAmounts = recorder.get().getAddressAmounts(new BlockHeightRange(begin, end)); + + if( addressAmounts.isPresent() ) { + return addressAmounts.get().getAmounts().stream() + .sorted(BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_COMPARATOR.reversed()) + .skip(offset) + .limit(limit) + .collect(Collectors.toList()); + } + else { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + } + else { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + } + @GET @Path("/openorders/{assetid}/{otherassetid}") @Operation( diff --git a/src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java b/src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java index 7a7009ff..43e7c542 100644 --- a/src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java +++ b/src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java @@ -2,15 +2,19 @@ package org.qortal.controller.hsqldb; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.util.PropertySource; import org.qortal.data.account.AccountBalanceData; +import org.qortal.data.account.BlockHeightRange; +import org.qortal.data.account.BlockHeightRangeAddressAmounts; import org.qortal.repository.hsqldb.HSQLDBCacheUtils; import org.qortal.settings.Settings; +import org.qortal.utils.BalanceRecorderUtils; -import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.stream.Collectors; public class HSQLDBBalanceRecorder extends Thread{ @@ -23,6 +27,8 @@ public class HSQLDBBalanceRecorder extends Thread{ private ConcurrentHashMap> balancesByAddress = new ConcurrentHashMap<>(); + private CopyOnWriteArrayList balanceDynamics = new CopyOnWriteArrayList<>(); + private int priorityRequested; private int frequency; private int capacity; @@ -61,36 +67,52 @@ public class HSQLDBBalanceRecorder extends Thread{ Thread.currentThread().setName("Balance Recorder"); - HSQLDBCacheUtils.startRecordingBalances(this.balancesByHeight, this.balancesByAddress, this.priorityRequested, this.frequency, this.capacity); + HSQLDBCacheUtils.startRecordingBalances(this.balancesByHeight, this.balanceDynamics, this.priorityRequested, this.frequency, this.capacity); } - public List getLatestRecordings(int limit, long offset) { - ArrayList data; + public List getLatestDynamics(int limit, long offset) { - Optional lastHeight = getLastHeight(); + List latest = this.balanceDynamics.stream() + .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.reversed()) + .skip(offset) + .limit(limit) + .collect(Collectors.toList()); - if(lastHeight.isPresent() ) { - List latest = this.balancesByHeight.get(lastHeight.get()); + return latest; + } - if( latest != null ) { - data = new ArrayList<>(latest.size()); - data.addAll( - latest.stream() - .sorted(Comparator.comparingDouble(AccountBalanceData::getBalance).reversed()) - .skip(offset) - .limit(limit) - .collect(Collectors.toList()) - ); - } - else { - data = new ArrayList<>(0); - } + public List getRanges(Integer offset, Integer limit, Boolean reverse) { + + if( reverse ) { + return this.balanceDynamics.stream() + .map(BlockHeightRangeAddressAmounts::getRange) + .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_COMPARATOR.reversed()) + .skip(offset) + .limit(limit) + .collect(Collectors.toList()); } else { - data = new ArrayList<>(0); + return this.balanceDynamics.stream() + .map(BlockHeightRangeAddressAmounts::getRange) + .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_COMPARATOR) + .skip(offset) + .limit(limit) + .collect(Collectors.toList()); } + } - return data; + public Optional getAddressAmounts(BlockHeightRange range) { + + return this.balanceDynamics.stream() + .filter( dynamic -> dynamic.getRange().equals(range)) + .findAny(); + } + + public Optional getRange( int height ) { + return this.balanceDynamics.stream() + .map(BlockHeightRangeAddressAmounts::getRange) + .filter( range -> range.getBegin() < height && range.getEnd() >= height ) + .findAny(); } private Optional getLastHeight() { diff --git a/src/main/java/org/qortal/data/account/AddressAmountData.java b/src/main/java/org/qortal/data/account/AddressAmountData.java new file mode 100644 index 00000000..d5a9f52b --- /dev/null +++ b/src/main/java/org/qortal/data/account/AddressAmountData.java @@ -0,0 +1,54 @@ +package org.qortal.data.account; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import java.util.Objects; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class AddressAmountData { + + private String address; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long amount; + + public AddressAmountData() { + } + + public AddressAmountData(String address, long amount) { + + this.address = address; + this.amount = amount; + } + + public String getAddress() { + return address; + } + + public long getAmount() { + return amount; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AddressAmountData that = (AddressAmountData) o; + return amount == that.amount && Objects.equals(address, that.address); + } + + @Override + public int hashCode() { + return Objects.hash(address, amount); + } + + @Override + public String toString() { + return "AddressAmountData{" + + "address='" + address + '\'' + + ", amount=" + amount + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/account/BlockHeightRange.java b/src/main/java/org/qortal/data/account/BlockHeightRange.java new file mode 100644 index 00000000..fd356cb3 --- /dev/null +++ b/src/main/java/org/qortal/data/account/BlockHeightRange.java @@ -0,0 +1,51 @@ +package org.qortal.data.account; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class BlockHeightRange { + + private int begin; + + private int end; + + public BlockHeightRange() { + } + + public BlockHeightRange(int begin, int end) { + this.begin = begin; + this.end = end; + } + + public int getBegin() { + return begin; + } + + public int getEnd() { + return end; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BlockHeightRange that = (BlockHeightRange) o; + return begin == that.begin && end == that.end; + } + + @Override + public int hashCode() { + return Objects.hash(begin, end); + } + + @Override + public String toString() { + return "BlockHeightRange{" + + "begin=" + begin + + ", end=" + end + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/account/BlockHeightRangeAddressAmounts.java b/src/main/java/org/qortal/data/account/BlockHeightRangeAddressAmounts.java new file mode 100644 index 00000000..3e2b7042 --- /dev/null +++ b/src/main/java/org/qortal/data/account/BlockHeightRangeAddressAmounts.java @@ -0,0 +1,52 @@ +package org.qortal.data.account; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.List; +import java.util.Objects; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class BlockHeightRangeAddressAmounts { + + private BlockHeightRange range; + + private List amounts; + + public BlockHeightRangeAddressAmounts() { + } + + public BlockHeightRangeAddressAmounts(BlockHeightRange range, List amounts) { + this.range = range; + this.amounts = amounts; + } + + public BlockHeightRange getRange() { + return range; + } + + public List getAmounts() { + return amounts; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BlockHeightRangeAddressAmounts that = (BlockHeightRangeAddressAmounts) o; + return Objects.equals(range, that.range) && Objects.equals(amounts, that.amounts); + } + + @Override + public int hashCode() { + return Objects.hash(range, amounts); + } + + @Override + public String toString() { + return "BlockHeightRangeAddressAmounts{" + + "range=" + range + + ", amounts=" + amounts + + '}'; + } +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java index a4aacbf5..f4d33b87 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java @@ -7,11 +7,16 @@ import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.Controller; import org.qortal.data.account.AccountBalanceData; +import org.qortal.data.account.AddressAmountData; +import org.qortal.data.account.BlockHeightRange; +import org.qortal.data.account.BlockHeightRangeAddressAmounts; import org.qortal.data.arbitrary.ArbitraryResourceCache; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.arbitrary.ArbitraryResourceStatus; import org.qortal.repository.DataException; +import org.qortal.settings.Settings; +import org.qortal.utils.BalanceRecorderUtils; import java.sql.ResultSet; import java.sql.SQLException; @@ -28,6 +33,7 @@ import java.util.Optional; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -389,14 +395,15 @@ public class HSQLDBCacheUtils { /** * Start Recording Balances * - * @param queue the queue to add to, remove oldest data if necssary - * @param repository the db repsoitory + * @param balancesByHeight height -> account balances + * @param balanceDynamics every balance dynamic * @param priorityRequested the requested thread priority - * @param frequency the recording frequencies, in minutes + * @param frequency the recording frequencies, in minutes + * @param capacity the maximum size of balanceDynamics */ public static void startRecordingBalances( final ConcurrentHashMap> balancesByHeight, - final ConcurrentHashMap> balancesByAddress, + CopyOnWriteArrayList balanceDynamics, int priorityRequested, int frequency, int capacity) { @@ -409,55 +416,64 @@ public class HSQLDBCacheUtils { Thread.currentThread().setName(BALANCE_RECORDER_TIMER_TASK); - try (final HSQLDBRepository repository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) { - while (balancesByHeight.size() > capacity + 1) { - Optional firstHeight = balancesByHeight.keySet().stream().sorted().findFirst(); + int currentHeight = recordCurrentBalances(balancesByHeight); - if (firstHeight.isPresent()) balancesByHeight.remove(firstHeight.get()); - } + LOGGER.debug("recorded balances: height = " + currentHeight); - // get current balances - List accountBalances = getAccountBalances(repository); + // remove invalidated recordings, recording after current height + BalanceRecorderUtils.removeRecordingsAboveHeight(currentHeight, balancesByHeight); - // get anyone of the balances - Optional data = accountBalances.stream().findAny(); + // remove invalidated dynamics, on or after current height + BalanceRecorderUtils.removeDynamicsOnOrAboveHeight(currentHeight, balanceDynamics); - // if there are any balances, then record them - if (data.isPresent()) { - // map all new balances to the current height - balancesByHeight.put(data.get().getHeight(), accountBalances); + // if there are 2 or more recordings, then produce balance dynamics for the first 2 recordings + if( balancesByHeight.size() > 1 ) { - // for each new balance, map to address - for (AccountBalanceData accountBalance : accountBalances) { + Optional priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight); - // get recorded balances for this address - List establishedBalances - = balancesByAddress.getOrDefault(accountBalance.getAddress(), new ArrayList<>(0)); + // if there is a prior height + if(priorHeight.isPresent()) { - // start a new list of recordings for this address, add the new balance and add the established - // balances - List balances = new ArrayList<>(establishedBalances.size() + 1); - balances.add(accountBalance); - balances.addAll(establishedBalances); + BlockHeightRange blockHeightRange = new BlockHeightRange(priorHeight.get(), currentHeight); - // reset tha balances for this address - balancesByAddress.put(accountBalance.getAddress(), balances); + LOGGER.debug("building dynamics for block heights: range = " + blockHeightRange); - // TODO: reduce account balances to capacity - } + List currentBalances = balancesByHeight.get(currentHeight); - // reduce height balances to capacity - while( balancesByHeight.size() > capacity ) { - Optional lowestHeight - = balancesByHeight.entrySet().stream() - .min(Comparator.comparingInt(Map.Entry::getKey)) - .map(Map.Entry::getKey); + List currentDynamics + = BalanceRecorderUtils.buildBalanceDynamics( + currentBalances, + balancesByHeight.get(priorHeight.get()), + Settings.getInstance().getMinimumBalanceRecording()); - if (lowestHeight.isPresent()) balancesByHeight.entrySet().remove(lowestHeight); + LOGGER.debug("dynamics built: count = " + currentDynamics.size()); + + if(LOGGER.isDebugEnabled()) + currentDynamics.stream() + .sorted(Comparator.comparingLong(AddressAmountData::getAmount).reversed()) + .limit(Settings.getInstance().getTopBalanceLoggingLimit()) + .forEach(top5Dynamic -> LOGGER.debug("Top Dynamics = " + top5Dynamic)); + + BlockHeightRangeAddressAmounts amounts + = new BlockHeightRangeAddressAmounts( blockHeightRange, currentDynamics ); + + balanceDynamics.add(amounts); + + BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight - Settings.getInstance().getBalanceRecorderRollbackAllowance(), balancesByHeight); + + while(balanceDynamics.size() > capacity) { + BlockHeightRangeAddressAmounts oldestDynamics = BalanceRecorderUtils.removeOldestDynamics(balanceDynamics); + + LOGGER.debug("removing oldest dynamics: range " + oldestDynamics.getRange()); } } - } catch (DataException e) { - LOGGER.error(e.getMessage(), e); + else { + LOGGER.warn("Expecting prior height and nothing was discovered, current height = " + currentHeight); + } + } + // else this should be the first recording + else { + LOGGER.info("first balance recording completed"); } } }; @@ -466,6 +482,35 @@ public class HSQLDBCacheUtils { timer.scheduleAtFixedRate(task, 300_000, frequency * 60_000); } + private static int recordCurrentBalances(ConcurrentHashMap> balancesByHeight) { + int currentHeight; + + try (final HSQLDBRepository repository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) { + + // get current balances + List accountBalances = getAccountBalances(repository); + + // get anyone of the balances + Optional data = accountBalances.stream().findAny(); + + // if there are any balances, then record them + if (data.isPresent()) { + // map all new balances to the current height + balancesByHeight.put(data.get().getHeight(), accountBalances); + + currentHeight = data.get().getHeight(); + } + else { + currentHeight = Integer.MAX_VALUE; + } + } catch (DataException e) { + LOGGER.error(e.getMessage(), e); + currentHeight = Integer.MAX_VALUE; + } + + return currentHeight; + } + /** * Build Timer * diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 2a749242..5222f22e 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -444,14 +444,56 @@ public class Settings { */ private long archivingPause = 3000; + /** + * Enable Balance Recorder? + * + * True for balance recording, otherwise false. + */ private boolean balanceRecorderEnabled = false; + /** + * Balance Recorder Priority + * + * The thread priority (1 is lowest, 10 is highest) of the balance recorder thread, if enabled. + */ private int balanceRecorderPriority = 1; - private int balanceRecorderFrequency = 2*60*1000; + /** + * Balance Recorder Frequency + * + * How often the balances will be recorded, if enabled, measured in minutes. + */ + private int balanceRecorderFrequency = 20; + /** + * Balance Recorder Capacity + * + * The number of balance recorder ranges will be held in memory. + */ private int balanceRecorderCapacity = 1000; + /** + * Minimum Balance Recording + * + * The minimum recored balance change in Qortoshis (1/100000000 QORT) + */ + private long minimumBalanceRecording = 100000000; + + /** + * Top Balance Logging Limit + * + * When logging the number limit of top balance changes to show in the logs for any given block range. + */ + private long topBalanceLoggingLimit = 100; + + /** + * Balance Recorder Rollback Allowance + * + * If the balance recorder is enabled, it must protect its prior balances by this number of blocks in case of + * a blockchain rollback and reorganization. + */ + private int balanceRecorderRollbackAllowance = 100; + // Domain mapping public static class ThreadLimit { private String messageType; @@ -1257,4 +1299,16 @@ public class Settings { public boolean isBalanceRecorderEnabled() { return balanceRecorderEnabled; } + + public long getMinimumBalanceRecording() { + return minimumBalanceRecording; + } + + public long getTopBalanceLoggingLimit() { + return topBalanceLoggingLimit; + } + + public int getBalanceRecorderRollbackAllowance() { + return balanceRecorderRollbackAllowance; + } } diff --git a/src/main/java/org/qortal/utils/BalanceRecorderUtils.java b/src/main/java/org/qortal/utils/BalanceRecorderUtils.java new file mode 100644 index 00000000..44d8f5e8 --- /dev/null +++ b/src/main/java/org/qortal/utils/BalanceRecorderUtils.java @@ -0,0 +1,119 @@ +package org.qortal.utils; + +import org.qortal.data.account.AccountBalanceData; +import org.qortal.data.account.AddressAmountData; +import org.qortal.data.account.BlockHeightRange; +import org.qortal.data.account.BlockHeightRangeAddressAmounts; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public class BalanceRecorderUtils { + + public static final Predicate ADDRESS_AMOUNT_DATA_NOT_ZERO = addressAmount -> addressAmount.getAmount() != 0; + public static final Comparator BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR = new Comparator() { + @Override + public int compare(BlockHeightRangeAddressAmounts amounts1, BlockHeightRangeAddressAmounts amounts2) { + return amounts1.getRange().getEnd() - amounts2.getRange().getEnd(); + } + }; + + public static final Comparator ADDRESS_AMOUNT_DATA_COMPARATOR = new Comparator() { + @Override + public int compare(AddressAmountData addressAmountData, AddressAmountData t1) { + if( addressAmountData.getAmount() > t1.getAmount() ) { + return 1; + } + else if( addressAmountData.getAmount() < t1.getAmount() ) { + return -1; + } + else { + return 0; + } + } + }; + + public static final Comparator BLOCK_HEIGHT_RANGE_COMPARATOR = new Comparator() { + @Override + public int compare(BlockHeightRange range1, BlockHeightRange range2) { + return range1.getEnd() - range2.getEnd(); + } + }; + + /** + * Build Balance Dynmaics For Account + * + * @param priorBalances the balances prior to the current height, assuming only one balance per address + * @param accountBalance the current balance + * + * @return the difference between the current balance and the prior balance for the current balance address + */ + public static AddressAmountData buildBalanceDynamicsForAccount(List priorBalances, AccountBalanceData accountBalance) { + Optional matchingAccountPriorBalance + = priorBalances.stream() + .filter(priorBalance -> accountBalance.getAddress().equals(priorBalance.getAddress())) + .findFirst(); + if(matchingAccountPriorBalance.isPresent()) { + return new AddressAmountData(accountBalance.getAddress(), accountBalance.getBalance() - matchingAccountPriorBalance.get().getBalance()); + } + else { + return new AddressAmountData(accountBalance.getAddress(), accountBalance.getBalance()); + } + } + + public static List buildBalanceDynamics(final List balances, final List priorBalances, long minimum) { + + List addressAmounts = new ArrayList<>(balances.size()); + + // prior balance + addressAmounts.addAll( + balances.stream() + .map(balance -> buildBalanceDynamicsForAccount(priorBalances, balance)) + .filter(ADDRESS_AMOUNT_DATA_NOT_ZERO) + .filter( data -> data.getAmount() >= minimum) + .collect(Collectors.toList()) + ); + + return addressAmounts; + } + + public static void removeRecordingsAboveHeight(int currentHeight, ConcurrentHashMap> balancesByHeight) { + balancesByHeight.entrySet().stream() + .filter(heightWithBalances -> heightWithBalances.getKey() > currentHeight) + .forEach(heightWithBalances -> balancesByHeight.remove(heightWithBalances.getKey())); + } + + public static void removeRecordingsBelowHeight(int currentHeight, ConcurrentHashMap> balancesByHeight) { + balancesByHeight.entrySet().stream() + .filter(heightWithBalances -> heightWithBalances.getKey() < currentHeight) + .forEach(heightWithBalances -> balancesByHeight.remove(heightWithBalances.getKey())); + } + + public static void removeDynamicsOnOrAboveHeight(int currentHeight, CopyOnWriteArrayList balanceDynamics) { + balanceDynamics.stream() + .filter(addressAmounts -> addressAmounts.getRange().getEnd() >= currentHeight) + .forEach(addressAmounts -> balanceDynamics.remove(addressAmounts)); + } + + public static BlockHeightRangeAddressAmounts removeOldestDynamics(CopyOnWriteArrayList balanceDynamics) { + BlockHeightRangeAddressAmounts oldestDynamics + = balanceDynamics.stream().sorted(BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR).findFirst().get(); + + balanceDynamics.remove(oldestDynamics); + return oldestDynamics; + } + + public static Optional getPriorHeight(int currentHeight, ConcurrentHashMap> balancesByHeight) { + Optional priorHeight + = balancesByHeight.keySet().stream() + .filter(height -> height < currentHeight) + .sorted(Comparator.reverseOrder()).findFirst(); + return priorHeight; + } +} diff --git a/src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java b/src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java new file mode 100644 index 00000000..746c845f --- /dev/null +++ b/src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java @@ -0,0 +1,361 @@ +package org.qortal.test.utils; + +import org.junit.Assert; +import org.junit.Test; +import org.qortal.data.account.AccountBalanceData; +import org.qortal.data.account.AddressAmountData; +import org.qortal.data.account.BlockHeightRange; +import org.qortal.data.account.BlockHeightRangeAddressAmounts; +import org.qortal.utils.BalanceRecorderUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; + +public class BalanceRecorderUtilsTests { + + @Test + public void testNotZeroForZero() { + boolean test = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_NOT_ZERO.test( new AddressAmountData("", 0)); + + Assert.assertFalse(test); + } + + @Test + public void testNotZeroForPositive() { + boolean test = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_NOT_ZERO.test(new AddressAmountData("", 1)); + + Assert.assertTrue(test); + } + + @Test + public void testNotZeroForNegative() { + boolean test = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_NOT_ZERO.test( new AddressAmountData("", -10)); + + Assert.assertTrue(test); + } + + @Test + public void testAddressAmountComparatorReverseOrder() { + + BlockHeightRangeAddressAmounts addressAmounts1 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(2, 3), new ArrayList<>(0)); + BlockHeightRangeAddressAmounts addressAmounts2 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 2), new ArrayList<>(0)); + + int compare = BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.compare(addressAmounts1, addressAmounts2); + + Assert.assertTrue( compare > 0); + } + + @Test + public void testAddressAmountComparatorForwardOrder() { + + BlockHeightRangeAddressAmounts addressAmounts1 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 2), new ArrayList<>(0)); + BlockHeightRangeAddressAmounts addressAmounts2 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(2, 3), new ArrayList<>(0)); + + int compare = BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.compare(addressAmounts1, addressAmounts2); + + Assert.assertTrue( compare < 0 ); + } + + @Test + public void testAddressAmountDataComparator() { + + AddressAmountData addressAmount1 = new AddressAmountData("a", 10); + AddressAmountData addressAmount2 = new AddressAmountData("b", 20); + + int compare = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_COMPARATOR.compare(addressAmount1, addressAmount2); + + Assert.assertTrue( compare < 0); + } + + @Test + public void testRemoveRecordingsBelowHeightNoBalances() { + + int currentHeight = 5; + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(); + + BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight, balancesByHeight); + + Assert.assertEquals(0, balancesByHeight.size()); + } + + @Test + public void testRemoveRecordingsBelowHeightOneBalanceBelow() { + int currentHeight = 5; + + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(1); + + balancesByHeight.put(1, new ArrayList<>(0)); + + Assert.assertEquals(1, balancesByHeight.size()); + + BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight, balancesByHeight); + + Assert.assertEquals(0, balancesByHeight.size()); + } + + @Test + public void testRemoveRecordingsBelowHeightOneBalanceAbove() { + int currentHeight = 5; + + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(1); + + balancesByHeight.put(10, new ArrayList<>(0)); + + Assert.assertEquals(1, balancesByHeight.size()); + + BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight, balancesByHeight); + + Assert.assertEquals(1, balancesByHeight.size()); + } + + @Test + public void testBuildBalanceDynamicsOneAccountOneChange() { + + String address = "a"; + + List balances = new ArrayList<>(1); + balances.add(new AccountBalanceData(address, 0, 2)); + + List priorBalances = new ArrayList<>(1); + priorBalances.add(new AccountBalanceData(address, 0, 1)); + + List dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, 0); + + Assert.assertNotNull(dynamics); + Assert.assertEquals(1, dynamics.size()); + + AddressAmountData addressAmountData = dynamics.get(0); + Assert.assertNotNull(addressAmountData); + Assert.assertEquals(address, addressAmountData.getAddress()); + Assert.assertEquals(1, addressAmountData.getAmount()); + } + + @Test + public void testBuildBalanceDynamicsOneAccountNoPrior() { + + String address = "a"; + + List balances = new ArrayList<>(1); + balances.add(new AccountBalanceData(address, 0, 2)); + + List priorBalances = new ArrayList<>(0); + + List dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, 0); + + Assert.assertNotNull(dynamics); + Assert.assertEquals(1, dynamics.size()); + + AddressAmountData addressAmountData = dynamics.get(0); + Assert.assertNotNull(addressAmountData); + Assert.assertEquals(address, addressAmountData.getAddress()); + Assert.assertEquals(2, addressAmountData.getAmount()); + } + + @Test + public void testBuildBalanceDynamicsTwoAccountsNegativeValues() { + + String address1 = "a"; + String address2 = "b"; + + List balances = new ArrayList<>(2); + balances.add(new AccountBalanceData(address1, 0, 10_000)); + balances.add(new AccountBalanceData(address2, 0, 100)); + + List priorBalances = new ArrayList<>(2); + priorBalances.add(new AccountBalanceData(address2, 0, 200)); + priorBalances.add(new AccountBalanceData(address1, 0, 5000)); + + List dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, -100L); + + Assert.assertNotNull(dynamics); + Assert.assertEquals(2, dynamics.size()); + + Map amountByAddress + = dynamics.stream() + .collect(Collectors.toMap(dynamic -> dynamic.getAddress(), dynamic -> dynamic.getAmount())); + + Assert.assertTrue(amountByAddress.containsKey(address1)); + + long amount1 = amountByAddress.get(address1); + + Assert.assertNotNull(amount1); + Assert.assertEquals(5000L, amount1 ); + + Assert.assertTrue(amountByAddress.containsKey(address2)); + + long amount2 = amountByAddress.get(address2); + + Assert.assertNotNull(amount2); + Assert.assertEquals(-100L, amount2); + } + + @Test + public void testBuildBalanceDynamicsForAccountNoPriorAnyAccount() { + List priorBalances = new ArrayList<>(0); + AccountBalanceData accountBalance = new AccountBalanceData("a", 0, 10); + + AddressAmountData dynamic = BalanceRecorderUtils.buildBalanceDynamicsForAccount(priorBalances, accountBalance); + + Assert.assertNotNull(dynamic); + Assert.assertEquals(10, dynamic.getAmount()); + Assert.assertEquals("a", dynamic.getAddress()); + } + + @Test + public void testBuildBalanceDynamicsForAccountNoPriorThisAccount() { + List priorBalances = new ArrayList<>(2); + priorBalances.add(new AccountBalanceData("b", 0, 100)); + + AccountBalanceData accountBalanceData = new AccountBalanceData("a", 0, 10); + + AddressAmountData dynamic = BalanceRecorderUtils.buildBalanceDynamicsForAccount(priorBalances, accountBalanceData); + + Assert.assertNotNull(dynamic); + Assert.assertEquals(10, dynamic.getAmount()); + Assert.assertEquals("a", dynamic.getAddress()); + } + + @Test + public void testBuildBalanceDynamicsForAccountPriorForThisAndOthers() { + List priorBalances = new ArrayList<>(2); + priorBalances.add(new AccountBalanceData("a", 0, 100)); + priorBalances.add(new AccountBalanceData("b", 0, 200)); + priorBalances.add(new AccountBalanceData("c", 0, 300)); + + AccountBalanceData accountBalance = new AccountBalanceData("b", 0, 1000); + + AddressAmountData dynamic = BalanceRecorderUtils.buildBalanceDynamicsForAccount(priorBalances, accountBalance); + + Assert.assertNotNull(dynamic); + Assert.assertEquals(800, dynamic.getAmount()); + Assert.assertEquals("b", dynamic.getAddress()); + } + + @Test + public void testRemoveRecordingAboveHeightOneOfTwo() { + + int currentHeight = 10; + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(); + + balancesByHeight.put(3, new ArrayList<>()); + balancesByHeight.put(20, new ArrayList<>()); + + Assert.assertEquals(2, balancesByHeight.size()); + + BalanceRecorderUtils.removeRecordingsAboveHeight(currentHeight, balancesByHeight); + + Assert.assertEquals(1, balancesByHeight.size()); + Assert.assertTrue( balancesByHeight.containsKey(3)); + } + + @Test + public void testPriorHeightBeforeAfter() { + + int currentHeight = 10; + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(); + balancesByHeight.put( 2, new ArrayList<>()); + balancesByHeight.put(7, new ArrayList<>()); + balancesByHeight.put(12, new ArrayList<>()); + + Optional priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight); + + Assert.assertNotNull(priorHeight); + Assert.assertTrue(priorHeight.isPresent()); + Assert.assertEquals( 7, priorHeight.get().intValue()); + } + + @Test + public void testPriorHeightNoPriorAfterOnly() { + + int currentHeight = 10; + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(); + balancesByHeight.put(12, new ArrayList<>()); + + Optional priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight); + + Assert.assertNotNull(priorHeight); + Assert.assertTrue(priorHeight.isEmpty()); + } + + @Test + public void testPriorHeightPriorOnly() { + + int currentHeight = 10; + + ConcurrentHashMap> balancesByHeight = new ConcurrentHashMap<>(); + balancesByHeight.put(7, new ArrayList<>()); + + Optional priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight); + + Assert.assertNotNull(priorHeight); + Assert.assertTrue(priorHeight.isPresent()); + Assert.assertEquals(7, priorHeight.get().intValue()); + } + + @Test + public void testRemoveDynamicsOnOrAboveHeightOneAbove() { + + int currentHeight = 10; + + CopyOnWriteArrayList dynamics = new CopyOnWriteArrayList<>(); + + BlockHeightRange range1 = new BlockHeightRange(10, 20); + dynamics.add(new BlockHeightRangeAddressAmounts(range1, new ArrayList<>())); + + BlockHeightRange range2 = new BlockHeightRange(1, 4); + dynamics.add(new BlockHeightRangeAddressAmounts(range2, new ArrayList<>())); + + Assert.assertEquals(2, dynamics.size()); + BalanceRecorderUtils.removeDynamicsOnOrAboveHeight(currentHeight, dynamics); + + Assert.assertEquals(1, dynamics.size()); + Assert.assertEquals(range2, dynamics.get(0).getRange()); + } + + @Test + public void testRemoveDynamicsOnOrAboveOneOnOneAbove() { + int currentHeight = 11; + + CopyOnWriteArrayList dynamics = new CopyOnWriteArrayList<>(); + + BlockHeightRange range1 = new BlockHeightRange(1,5); + dynamics.add(new BlockHeightRangeAddressAmounts(range1, new ArrayList<>())); + + BlockHeightRange range2 = new BlockHeightRange(6, 11); + dynamics.add((new BlockHeightRangeAddressAmounts(range2, new ArrayList<>()))); + + BlockHeightRange range3 = new BlockHeightRange(22, 16); + dynamics.add(new BlockHeightRangeAddressAmounts(range3, new ArrayList<>())); + + Assert.assertEquals(3, dynamics.size()); + + BalanceRecorderUtils.removeDynamicsOnOrAboveHeight(currentHeight, dynamics); + + Assert.assertEquals(1, dynamics.size()); + Assert.assertTrue( dynamics.get(0).getRange().equals(range1)); + } + + @Test + public void testRemoveOldestDynamicsTwice() { + CopyOnWriteArrayList dynamics = new CopyOnWriteArrayList<>(); + + dynamics.add(new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 5), new ArrayList<>())); + dynamics.add(new BlockHeightRangeAddressAmounts(new BlockHeightRange(5, 9), new ArrayList<>())); + + Assert.assertEquals(2, dynamics.size()); + + BalanceRecorderUtils.removeOldestDynamics(dynamics); + + Assert.assertEquals(1, dynamics.size()); + Assert.assertTrue(dynamics.get(0).getRange().equals(new BlockHeightRange(5, 9))); + + BalanceRecorderUtils.removeOldestDynamics(dynamics); + + Assert.assertEquals(0, dynamics.size()); + } +} \ No newline at end of file From c71f5fa8bf581844ce56855dab38530fb9970a5b Mon Sep 17 00:00:00 2001 From: kennycud Date: Fri, 13 Dec 2024 15:21:51 -0800 Subject: [PATCH 03/24] added another logging line to troubleshoot QDN problem --- src/main/java/org/qortal/transaction/ArbitraryTransaction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index e5fa4f58..ee9b0b8e 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -388,6 +388,7 @@ public class ArbitraryTransaction extends Transaction { // Get the latest transaction ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); if (latestTransactionData == null) { + LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData); // We don't have a latest transaction, so delete from cache repository.getArbitraryRepository().delete(arbitraryResourceData); return; From e9b75b051b44b503efdb8f19886a4dff39c8be94 Mon Sep 17 00:00:00 2001 From: PhilReact Date: Tue, 24 Dec 2024 14:39:31 +0200 Subject: [PATCH 04/24] added seller/buyer to filter completed trades --- .../api/resource/CrossChainResource.java | 10 +++++- .../api/websocket/TradeOffersWebSocket.java | 6 ++-- .../org/qortal/repository/ATRepository.java | 2 +- .../repository/hsqldb/HSQLDBATRepository.java | 33 +++++++++++++++---- .../java/org/qortal/test/RepositoryTests.java | 2 +- .../qortal/test/api/CrossChainApiTests.java | 6 ++-- .../org/qortal/test/at/AtRepositoryTests.java | 4 +++ 7 files changed, 48 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/CrossChainResource.java b/src/main/java/org/qortal/api/resource/CrossChainResource.java index 9e411127..748dcbe4 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainResource.java @@ -52,6 +52,8 @@ import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; + + @Path("/crosschain") @Tag(name = "Cross-Chain") public class CrossChainResource { @@ -255,6 +257,12 @@ public class CrossChainResource { description = "Only return trades that completed on/after this timestamp (milliseconds since epoch)", example = "1597310000000" ) @QueryParam("minimumTimestamp") Long minimumTimestamp, + @Parameter( + description = "Optionally filter by buyer Qortal address" + ) @QueryParam("buyerAddress") String buyerAddress, + @Parameter( + description = "Optionally filter by seller Qortal address" + ) @QueryParam("sellerAddress") String sellerAddress, @Parameter( ref = "limit") @QueryParam("limit") Integer limit, @Parameter( ref = "offset" ) @QueryParam("offset") Integer offset, @Parameter( ref = "reverse" ) @QueryParam("reverse") Boolean reverse) { @@ -296,7 +304,7 @@ public class CrossChainResource { byte[] codeHash = acctInfo.getKey().value; ACCT acct = acctInfo.getValue().get(); - List atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, + List atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, buyerAddress, sellerAddress, isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumFinalHeight, limit, offset, reverse); diff --git a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java index 96257f4a..911cf188 100644 --- a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java @@ -98,7 +98,7 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener { byte[] codeHash = acctInfo.getKey().value; ACCT acct = acctInfo.getValue().get(); - List atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, + List atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, null, null, isFinished, dataByteOffset, expectedValue, minimumFinalHeight, null, null, null); @@ -259,7 +259,7 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener { ACCT acct = acctInfo.getValue().get(); Integer dataByteOffset = acct.getModeByteOffset(); - List initialAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash, + List initialAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash, null, null, isFinished, dataByteOffset, expectedValue, minimumFinalHeight, null, null, null); @@ -298,7 +298,7 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener { byte[] codeHash = acctInfo.getKey().value; ACCT acct = acctInfo.getValue().get(); - List historicAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash, + List historicAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash, null, null, isFinished, dataByteOffset, expectedValue, minimumFinalHeight, null, null, null); diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 455ba393..fe001137 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -76,7 +76,7 @@ public interface ATRepository { * Although expectedValue, if provided, is natively an unsigned long, * the data segment comparison is done via unsigned hex string. */ - public List getMatchingFinalATStates(byte[] codeHash, Boolean isFinished, + public List getMatchingFinalATStates(byte[] codeHash, String buyerAddress, String sellerAddress, Boolean isFinished, Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight, Integer limit, Integer offset, Boolean reverse) throws DataException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index 80fc62dc..71a95428 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -1,6 +1,7 @@ package org.qortal.repository.hsqldb; import com.google.common.primitives.Longs; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; @@ -16,6 +17,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import org.qortal.data.account.AccountData; + public class HSQLDBATRepository implements ATRepository { private static final Logger LOGGER = LogManager.getLogger(HSQLDBATRepository.class); @@ -400,7 +403,7 @@ public class HSQLDBATRepository implements ATRepository { } @Override - public List getMatchingFinalATStates(byte[] codeHash, Boolean isFinished, + public List getMatchingFinalATStates(byte[] codeHash, String buyerAddress, String sellerAddress, Boolean isFinished, Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(1024); @@ -421,10 +424,14 @@ public class HSQLDBATRepository implements ATRepository { // Order by AT_address and height to use compound primary key as index // Both must be the same direction (DESC) also - sql.append("ORDER BY ATStates.AT_address DESC, ATStates.height DESC " - + "LIMIT 1 " - + ") AS FinalATStates " - + "WHERE code_hash = ? "); + sql.append("ORDER BY ATStates.height DESC LIMIT 1) AS FinalATStates "); + + // Optional LEFT JOIN with ATTRANSACTIONS for buyerAddress + if (buyerAddress != null && !buyerAddress.isEmpty()) { + sql.append("LEFT JOIN ATTRANSACTIONS tx ON tx.at_address = ATs.AT_address "); + } + + sql.append("WHERE ATs.code_hash = ? "); bindParams.add(codeHash); if (isFinished != null) { @@ -443,6 +450,20 @@ public class HSQLDBATRepository implements ATRepository { bindParams.add(rawExpectedValue); } + if (buyerAddress != null && !buyerAddress.isEmpty()) { + sql.append("AND tx.recipient = ? "); + bindParams.add(buyerAddress); + } + + + if (sellerAddress != null && !sellerAddress.isEmpty()) { + // Convert sellerAddress to publicKey (method depends on your implementation) + AccountData accountData = this.repository.getAccountRepository().getAccount(sellerAddress); + byte[] publicKey = accountData.getPublicKey(); + sql.append("AND ATs.creator = ? "); + bindParams.add(publicKey); + } + sql.append(" ORDER BY FinalATStates.height "); if (reverse != null && reverse) sql.append("DESC"); @@ -483,7 +504,7 @@ public class HSQLDBATRepository implements ATRepository { Integer dataByteOffset, Long expectedValue, int minimumCount, int maximumCount, long minimumPeriod) throws DataException { // We need most recent entry first so we can use its timestamp to slice further results - List mostRecentStates = this.getMatchingFinalATStates(codeHash, isFinished, + List mostRecentStates = this.getMatchingFinalATStates(codeHash, null, null, isFinished, dataByteOffset, expectedValue, null, 1, 0, true); diff --git a/src/test/java/org/qortal/test/RepositoryTests.java b/src/test/java/org/qortal/test/RepositoryTests.java index 1b0a0e52..51d2535e 100644 --- a/src/test/java/org/qortal/test/RepositoryTests.java +++ b/src/test/java/org/qortal/test/RepositoryTests.java @@ -405,7 +405,7 @@ public class RepositoryTests extends Common { Integer offset = null; Boolean reverse = null; - hsqldb.getATRepository().getMatchingFinalATStates(codeHash, isFinished, dataByteOffset, expectedValue, minimumFinalHeight, limit, offset, reverse); + hsqldb.getATRepository().getMatchingFinalATStates(codeHash,null, null, isFinished, dataByteOffset, expectedValue, minimumFinalHeight, limit, offset, reverse); } catch (DataException e) { fail("HSQLDB bug #1580"); } diff --git a/src/test/java/org/qortal/test/api/CrossChainApiTests.java b/src/test/java/org/qortal/test/api/CrossChainApiTests.java index d4f25bce..e70193b2 100644 --- a/src/test/java/org/qortal/test/api/CrossChainApiTests.java +++ b/src/test/java/org/qortal/test/api/CrossChainApiTests.java @@ -26,7 +26,7 @@ public class CrossChainApiTests extends ApiCommon { @Test public void testGetCompletedTrades() { long minimumTimestamp = System.currentTimeMillis(); - assertNoApiError((limit, offset, reverse) -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, minimumTimestamp, limit, offset, reverse)); + assertNoApiError((limit, offset, reverse) -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, minimumTimestamp, null, null, limit, offset, reverse)); } @Test @@ -35,8 +35,8 @@ public class CrossChainApiTests extends ApiCommon { Integer offset = null; Boolean reverse = null; - assertApiError(ApiError.INVALID_CRITERIA, () -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, -1L /*minimumTimestamp*/, limit, offset, reverse)); - assertApiError(ApiError.INVALID_CRITERIA, () -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, 0L /*minimumTimestamp*/, limit, offset, reverse)); + assertApiError(ApiError.INVALID_CRITERIA, () -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, -1L /*minimumTimestamp*/, null, null, limit, offset, reverse)); + assertApiError(ApiError.INVALID_CRITERIA, () -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, 0L /*minimumTimestamp*/, null, null, limit, offset, reverse)); } } diff --git a/src/test/java/org/qortal/test/at/AtRepositoryTests.java b/src/test/java/org/qortal/test/at/AtRepositoryTests.java index 5472fdb8..5ee74497 100644 --- a/src/test/java/org/qortal/test/at/AtRepositoryTests.java +++ b/src/test/java/org/qortal/test/at/AtRepositoryTests.java @@ -218,6 +218,8 @@ public class AtRepositoryTests extends Common { List atStates = repository.getATRepository().getMatchingFinalATStates( codeHash, + null, + null, isFinished, dataByteOffset, expectedValue, @@ -264,6 +266,8 @@ public class AtRepositoryTests extends Common { List atStates = repository.getATRepository().getMatchingFinalATStates( codeHash, + null, + null, isFinished, dataByteOffset, expectedValue, From 5ac0027b5abd28b0f24caba431ca55aa958b3b6d Mon Sep 17 00:00:00 2001 From: PhilReact Date: Wed, 25 Dec 2024 09:16:35 +0200 Subject: [PATCH 05/24] fix css for qdn resource loading --- src/main/resources/loading/index.html | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/resources/loading/index.html b/src/main/resources/loading/index.html index 574645cc..24ff2ae3 100644 --- a/src/main/resources/loading/index.html +++ b/src/main/resources/loading/index.html @@ -20,17 +20,21 @@ width: 100%; text-align: center; z-index: 1000; - top: 45%; + top: 50%; -ms-transform: translateY(-50%); - transform: translateY(-50%); + transform: translate(-50% , -50%); + left: 50%; } #panel { text-align: center; background: white; + word-wrap: break-word; width: 350px; + max-width: 100%; margin: auto; padding: 25px; border-radius: 30px; + box-sizing: border-box; } #status { color: #03a9f4; From 4f0aabfb363b447bd9ae528eb6918b376703fb55 Mon Sep 17 00:00:00 2001 From: kennycud Date: Wed, 25 Dec 2024 13:24:24 -0800 Subject: [PATCH 06/24] For Balance Recorder, reward recordings only, that is the default. --- .../qortal/api/resource/AssetsResource.java | 2 +- .../qortal/data/account/BlockHeightRange.java | 10 +- .../repository/hsqldb/HSQLDBCacheUtils.java | 101 ++-- .../java/org/qortal/settings/Settings.java | 13 +- .../qortal/utils/BalanceRecorderUtils.java | 218 ++++++++- .../test/utils/BalanceRecorderUtilsTests.java | 432 +++++++++++++++++- .../java/org/qortal/test/utils/TestUtils.java | 48 ++ 7 files changed, 767 insertions(+), 57 deletions(-) create mode 100644 src/test/java/org/qortal/test/utils/TestUtils.java diff --git a/src/main/java/org/qortal/api/resource/AssetsResource.java b/src/main/java/org/qortal/api/resource/AssetsResource.java index 6de01434..49ed251a 100644 --- a/src/main/java/org/qortal/api/resource/AssetsResource.java +++ b/src/main/java/org/qortal/api/resource/AssetsResource.java @@ -283,7 +283,7 @@ public class AssetsResource { Optional recorder = HSQLDBBalanceRecorder.getInstance(); if( recorder.isPresent()) { - Optional addressAmounts = recorder.get().getAddressAmounts(new BlockHeightRange(begin, end)); + Optional addressAmounts = recorder.get().getAddressAmounts(new BlockHeightRange(begin, end, false)); if( addressAmounts.isPresent() ) { return addressAmounts.get().getAmounts().stream() diff --git a/src/main/java/org/qortal/data/account/BlockHeightRange.java b/src/main/java/org/qortal/data/account/BlockHeightRange.java index fd356cb3..0ddb60bf 100644 --- a/src/main/java/org/qortal/data/account/BlockHeightRange.java +++ b/src/main/java/org/qortal/data/account/BlockHeightRange.java @@ -12,12 +12,15 @@ public class BlockHeightRange { private int end; + private boolean isRewardDistribution; + public BlockHeightRange() { } - public BlockHeightRange(int begin, int end) { + public BlockHeightRange(int begin, int end, boolean isRewardDistribution) { this.begin = begin; this.end = end; + this.isRewardDistribution = isRewardDistribution; } public int getBegin() { @@ -28,6 +31,10 @@ public class BlockHeightRange { return end; } + public boolean isRewardDistribution() { + return isRewardDistribution; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -46,6 +53,7 @@ public class BlockHeightRange { return "BlockHeightRange{" + "begin=" + begin + ", end=" + end + + ", isRewardDistribution=" + isRewardDistribution + '}'; } } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java index f4d33b87..6e8dc8a8 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java @@ -3,6 +3,7 @@ package org.qortal.repository.hsqldb; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.api.SearchMode; +import org.qortal.api.resource.TransactionsResource; import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.Controller; @@ -14,7 +15,10 @@ import org.qortal.data.arbitrary.ArbitraryResourceCache; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.arbitrary.ArbitraryResourceStatus; +import org.qortal.data.transaction.TransactionData; import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.utils.BalanceRecorderUtils; @@ -434,37 +438,11 @@ public class HSQLDBCacheUtils { // if there is a prior height if(priorHeight.isPresent()) { - BlockHeightRange blockHeightRange = new BlockHeightRange(priorHeight.get(), currentHeight); + boolean isRewardDistribution = BalanceRecorderUtils.isRewardDistributionRange(priorHeight.get(), currentHeight); - LOGGER.debug("building dynamics for block heights: range = " + blockHeightRange); - - List currentBalances = balancesByHeight.get(currentHeight); - - List currentDynamics - = BalanceRecorderUtils.buildBalanceDynamics( - currentBalances, - balancesByHeight.get(priorHeight.get()), - Settings.getInstance().getMinimumBalanceRecording()); - - LOGGER.debug("dynamics built: count = " + currentDynamics.size()); - - if(LOGGER.isDebugEnabled()) - currentDynamics.stream() - .sorted(Comparator.comparingLong(AddressAmountData::getAmount).reversed()) - .limit(Settings.getInstance().getTopBalanceLoggingLimit()) - .forEach(top5Dynamic -> LOGGER.debug("Top Dynamics = " + top5Dynamic)); - - BlockHeightRangeAddressAmounts amounts - = new BlockHeightRangeAddressAmounts( blockHeightRange, currentDynamics ); - - balanceDynamics.add(amounts); - - BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight - Settings.getInstance().getBalanceRecorderRollbackAllowance(), balancesByHeight); - - while(balanceDynamics.size() > capacity) { - BlockHeightRangeAddressAmounts oldestDynamics = BalanceRecorderUtils.removeOldestDynamics(balanceDynamics); - - LOGGER.debug("removing oldest dynamics: range " + oldestDynamics.getRange()); + // if this range has a reward recording block or if other blocks are enabled for recording + if( isRewardDistribution || !Settings.getInstance().isRewardRecordingOnly() ) { + produceBalanceDynamics(currentHeight, priorHeight, isRewardDistribution, balancesByHeight, balanceDynamics, capacity); } } else { @@ -482,6 +460,69 @@ public class HSQLDBCacheUtils { timer.scheduleAtFixedRate(task, 300_000, frequency * 60_000); } + private static void produceBalanceDynamics(int currentHeight, Optional priorHeight, boolean isRewardDistribution, ConcurrentHashMap> balancesByHeight, CopyOnWriteArrayList balanceDynamics, int capacity) { + BlockHeightRange blockHeightRange = new BlockHeightRange(priorHeight.get(), currentHeight, isRewardDistribution); + + LOGGER.debug("building dynamics for block heights: range = " + blockHeightRange); + + List currentBalances = balancesByHeight.get(currentHeight); + + ArrayList transactions = getTransactionDataForBlocks(blockHeightRange); + + LOGGER.info("transactions counted for balance adjustments: count = " + transactions.size()); + List currentDynamics + = BalanceRecorderUtils.buildBalanceDynamics( + currentBalances, + balancesByHeight.get(priorHeight.get()), + Settings.getInstance().getMinimumBalanceRecording(), + transactions); + + LOGGER.debug("dynamics built: count = " + currentDynamics.size()); + + if(LOGGER.isDebugEnabled()) + currentDynamics.stream() + .sorted(Comparator.comparingLong(AddressAmountData::getAmount).reversed()) + .limit(Settings.getInstance().getTopBalanceLoggingLimit()) + .forEach(top5Dynamic -> LOGGER.debug("Top Dynamics = " + top5Dynamic)); + + BlockHeightRangeAddressAmounts amounts + = new BlockHeightRangeAddressAmounts( blockHeightRange, currentDynamics ); + + balanceDynamics.add(amounts); + + BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight - Settings.getInstance().getBalanceRecorderRollbackAllowance(), balancesByHeight); + + while(balanceDynamics.size() > capacity) { + BlockHeightRangeAddressAmounts oldestDynamics = BalanceRecorderUtils.removeOldestDynamics(balanceDynamics); + + LOGGER.debug("removing oldest dynamics: range " + oldestDynamics.getRange()); + } + } + + private static ArrayList getTransactionDataForBlocks(BlockHeightRange blockHeightRange) { + ArrayList transactions; + + try (final Repository repository = RepositoryManager.getRepository()) { + List signatures + = repository.getTransactionRepository().getSignaturesMatchingCriteria( + blockHeightRange.getBegin() + 1, blockHeightRange.getEnd() - blockHeightRange.getBegin(), + null, null,null, null, null, + TransactionsResource.ConfirmationStatus.CONFIRMED, + null, null, null); + + transactions = new ArrayList<>(signatures.size()); + for (byte[] signature : signatures) { + transactions.add(repository.getTransactionRepository().fromSignature(signature)); + } + + LOGGER.debug(String.format("Found %s transactions for " + blockHeightRange, transactions.size())); + } catch (Exception e) { + transactions = new ArrayList<>(0); + LOGGER.warn("Problems getting transactions for balance recording: " + e.getMessage()); + } + return transactions; + } + private static int recordCurrentBalances(ConcurrentHashMap> balancesByHeight) { int currentHeight; diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 5222f22e..deee0075 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -494,7 +494,14 @@ public class Settings { */ private int balanceRecorderRollbackAllowance = 100; - // Domain mapping + /** + * Is Reward Recording Only + * + * Set true to only retain the recordings that cover reward distributions, otherwise set false. + */ + private boolean rewardRecordingOnly = true; + + // Domain mapping public static class ThreadLimit { private String messageType; private Integer limit; @@ -1311,4 +1318,8 @@ public class Settings { public int getBalanceRecorderRollbackAllowance() { return balanceRecorderRollbackAllowance; } + + public boolean isRewardRecordingOnly() { + return rewardRecordingOnly; + } } diff --git a/src/main/java/org/qortal/utils/BalanceRecorderUtils.java b/src/main/java/org/qortal/utils/BalanceRecorderUtils.java index 44d8f5e8..8ad346ac 100644 --- a/src/main/java/org/qortal/utils/BalanceRecorderUtils.java +++ b/src/main/java/org/qortal/utils/BalanceRecorderUtils.java @@ -1,13 +1,26 @@ package org.qortal.utils; +import org.qortal.block.Block; +import org.qortal.crypto.Crypto; +import org.qortal.data.PaymentData; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AddressAmountData; import org.qortal.data.account.BlockHeightRange; import org.qortal.data.account.BlockHeightRangeAddressAmounts; +import org.qortal.data.transaction.ATTransactionData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.BuyNameTransactionData; +import org.qortal.data.transaction.CreateAssetOrderTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MultiPaymentTransactionData; +import org.qortal.data.transaction.PaymentTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.data.transaction.TransferAssetTransactionData; -import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -67,22 +80,190 @@ public class BalanceRecorderUtils { } } - public static List buildBalanceDynamics(final List balances, final List priorBalances, long minimum) { + public static List buildBalanceDynamics( + final List balances, + final List priorBalances, + long minimum, + List transactions) { - List addressAmounts = new ArrayList<>(balances.size()); + Map amountsByAddress = new HashMap<>(transactions.size()); - // prior balance - addressAmounts.addAll( - balances.stream() + for( TransactionData transactionData : transactions ) { + + mapBalanceModificationsForTransaction(amountsByAddress, transactionData); + } + + List addressAmounts + = balances.stream() .map(balance -> buildBalanceDynamicsForAccount(priorBalances, balance)) + .map( data -> adjustAddressAmount(amountsByAddress.getOrDefault(data.getAddress(), 0L), data)) .filter(ADDRESS_AMOUNT_DATA_NOT_ZERO) - .filter( data -> data.getAmount() >= minimum) - .collect(Collectors.toList()) - ); + .filter(data -> data.getAmount() >= minimum) + .collect(Collectors.toList()); return addressAmounts; } + public static AddressAmountData adjustAddressAmount(long adjustment, AddressAmountData data) { + + return new AddressAmountData(data.getAddress(), data.getAmount() - adjustment); + } + + public static void mapBalanceModificationsForTransaction(Map amountsByAddress, TransactionData transactionData) { + String creatorAddress; + + // AT Transaction + if( transactionData instanceof ATTransactionData) { + creatorAddress = mapBalanceModificationsForAtTransaction(amountsByAddress, (ATTransactionData) transactionData); + } + // Buy Name Transaction + else if( transactionData instanceof BuyNameTransactionData) { + creatorAddress = mapBalanceModificationsForBuyNameTransaction(amountsByAddress, (BuyNameTransactionData) transactionData); + } + // Create Asset Order Transaction + else if( transactionData instanceof CreateAssetOrderTransactionData) { + //TODO I'm not sure how to handle this one. This hasn't been used at this point in the blockchain. + + creatorAddress = Crypto.toAddress(transactionData.getCreatorPublicKey()); + } + // Deploy AT Transaction + else if( transactionData instanceof DeployAtTransactionData ) { + creatorAddress = mapBalanceModificationsForDeployAtTransaction(amountsByAddress, (DeployAtTransactionData) transactionData); + } + // Multi Payment Transaction + else if( transactionData instanceof MultiPaymentTransactionData) { + creatorAddress = mapBalanceModificationsForMultiPaymentTransaction(amountsByAddress, (MultiPaymentTransactionData) transactionData); + } + // Payment Transaction + else if( transactionData instanceof PaymentTransactionData ) { + creatorAddress = mapBalanceModicationsForPaymentTransaction(amountsByAddress, (PaymentTransactionData) transactionData); + } + // Transfer Asset Transaction + else if( transactionData instanceof TransferAssetTransactionData) { + creatorAddress = mapBalanceModificationsForTransferAssetTransaction(amountsByAddress, (TransferAssetTransactionData) transactionData); + } + // Other Transactions + else { + creatorAddress = Crypto.toAddress(transactionData.getCreatorPublicKey()); + } + + // all transactions modify the balance for fees + mapBalanceModifications(amountsByAddress, transactionData.getFee(), creatorAddress, Optional.empty()); + } + + public static String mapBalanceModificationsForTransferAssetTransaction(Map amountsByAddress, TransferAssetTransactionData transferAssetData) { + String creatorAddress = Crypto.toAddress(transferAssetData.getSenderPublicKey()); + + if( transferAssetData.getAssetId() == 0) { + mapBalanceModifications( + amountsByAddress, + transferAssetData.getAmount(), + creatorAddress, + Optional.of(transferAssetData.getRecipient()) + ); + } + return creatorAddress; + } + + public static String mapBalanceModicationsForPaymentTransaction(Map amountsByAddress, PaymentTransactionData paymentData) { + String creatorAddress = Crypto.toAddress(paymentData.getCreatorPublicKey()); + + mapBalanceModifications(amountsByAddress, + paymentData.getAmount(), + creatorAddress, + Optional.of(paymentData.getRecipient()) + ); + return creatorAddress; + } + + public static String mapBalanceModificationsForMultiPaymentTransaction(Map amountsByAddress, MultiPaymentTransactionData multiPaymentData) { + String creatorAddress = Crypto.toAddress(multiPaymentData.getCreatorPublicKey()); + + for(PaymentData payment : multiPaymentData.getPayments() ) { + mapBalanceModificationsForTransaction( + amountsByAddress, + getPaymentTransactionData(multiPaymentData, payment) + ); + } + return creatorAddress; + } + + public static String mapBalanceModificationsForDeployAtTransaction(Map amountsByAddress, DeployAtTransactionData transactionData) { + String creatorAddress; + DeployAtTransactionData deployAtData = transactionData; + + creatorAddress = Crypto.toAddress(deployAtData.getCreatorPublicKey()); + + if( deployAtData.getAssetId() == 0 ) { + mapBalanceModifications( + amountsByAddress, + deployAtData.getAmount(), + creatorAddress, + Optional.of(deployAtData.getAtAddress()) + ); + } + return creatorAddress; + } + + public static String mapBalanceModificationsForBuyNameTransaction(Map amountsByAddress, BuyNameTransactionData transactionData) { + String creatorAddress; + BuyNameTransactionData buyNameData = transactionData; + + creatorAddress = Crypto.toAddress(buyNameData.getCreatorPublicKey()); + + mapBalanceModifications( + amountsByAddress, + buyNameData.getAmount(), + creatorAddress, + Optional.of(buyNameData.getSeller()) + ); + return creatorAddress; + } + + public static String mapBalanceModificationsForAtTransaction(Map amountsByAddress, ATTransactionData transactionData) { + String creatorAddress; + ATTransactionData atData = transactionData; + creatorAddress = atData.getATAddress(); + + if( atData.getAssetId() != null && atData.getAssetId() == 0) { + mapBalanceModifications( + amountsByAddress, + atData.getAmount(), + creatorAddress, + Optional.of(atData.getRecipient()) + ); + } + return creatorAddress; + } + + public static PaymentTransactionData getPaymentTransactionData(MultiPaymentTransactionData multiPaymentData, PaymentData payment) { + return new PaymentTransactionData( + new BaseTransactionData( + multiPaymentData.getTimestamp(), + multiPaymentData.getTxGroupId(), + multiPaymentData.getReference(), + multiPaymentData.getCreatorPublicKey(), + 0L, + multiPaymentData.getSignature() + ), + payment.getRecipient(), + payment.getAmount() + ); + } + + public static void mapBalanceModifications(Map amountsByAddress, Long amount, String sender, Optional recipient) { + amountsByAddress.put( + sender, + amountsByAddress.getOrDefault(sender, 0L) - amount + ); + + if( recipient.isPresent() ) + amountsByAddress.put( + recipient.get(), + amountsByAddress.getOrDefault(recipient.get(), 0L) + amount + ); + } + public static void removeRecordingsAboveHeight(int currentHeight, ConcurrentHashMap> balancesByHeight) { balancesByHeight.entrySet().stream() .filter(heightWithBalances -> heightWithBalances.getKey() > currentHeight) @@ -116,4 +297,23 @@ public class BalanceRecorderUtils { .sorted(Comparator.reverseOrder()).findFirst(); return priorHeight; } + + /** + * Is Reward Distribution Range? + * + * @param start start height, exclusive + * @param end end height, inclusive + * + * @return true there is a reward distribution block within this block range + */ + public static boolean isRewardDistributionRange(int start, int end) { + + // iterate through the block height until a reward distribution block or the end of the range + for( int i = start + 1; i <= end; i++) { + if( Block.isRewardDistributionBlock(i) ) return true; + } + + // no reward distribution blocks found within range + return false; + } } diff --git a/src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java b/src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java index 746c845f..0a35657a 100644 --- a/src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java +++ b/src/test/java/org/qortal/test/utils/BalanceRecorderUtilsTests.java @@ -2,13 +2,27 @@ package org.qortal.test.utils; import org.junit.Assert; import org.junit.Test; +import org.qortal.asset.Asset; +import org.qortal.block.Block; +import org.qortal.crypto.Crypto; +import org.qortal.data.PaymentData; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AddressAmountData; import org.qortal.data.account.BlockHeightRange; import org.qortal.data.account.BlockHeightRangeAddressAmounts; +import org.qortal.data.transaction.ATTransactionData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.BuyNameTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MultiPaymentTransactionData; +import org.qortal.data.transaction.PaymentTransactionData; +import org.qortal.data.transaction.RegisterNameTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.data.transaction.TransferAssetTransactionData; import org.qortal.utils.BalanceRecorderUtils; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -18,6 +32,10 @@ import java.util.stream.Collectors; public class BalanceRecorderUtilsTests { + public static final String RECIPIENT_ADDRESS = "recipient"; + public static final String AT_ADDRESS = "atAddress"; + public static final String OTHER = "Other"; + @Test public void testNotZeroForZero() { boolean test = BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_NOT_ZERO.test( new AddressAmountData("", 0)); @@ -42,8 +60,8 @@ public class BalanceRecorderUtilsTests { @Test public void testAddressAmountComparatorReverseOrder() { - BlockHeightRangeAddressAmounts addressAmounts1 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(2, 3), new ArrayList<>(0)); - BlockHeightRangeAddressAmounts addressAmounts2 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 2), new ArrayList<>(0)); + BlockHeightRangeAddressAmounts addressAmounts1 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(2, 3, false), new ArrayList<>(0)); + BlockHeightRangeAddressAmounts addressAmounts2 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 2, false), new ArrayList<>(0)); int compare = BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.compare(addressAmounts1, addressAmounts2); @@ -53,8 +71,8 @@ public class BalanceRecorderUtilsTests { @Test public void testAddressAmountComparatorForwardOrder() { - BlockHeightRangeAddressAmounts addressAmounts1 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 2), new ArrayList<>(0)); - BlockHeightRangeAddressAmounts addressAmounts2 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(2, 3), new ArrayList<>(0)); + BlockHeightRangeAddressAmounts addressAmounts1 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 2, false), new ArrayList<>(0)); + BlockHeightRangeAddressAmounts addressAmounts2 = new BlockHeightRangeAddressAmounts(new BlockHeightRange(2, 3, false), new ArrayList<>(0)); int compare = BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.compare(addressAmounts1, addressAmounts2); @@ -124,7 +142,7 @@ public class BalanceRecorderUtilsTests { List priorBalances = new ArrayList<>(1); priorBalances.add(new AccountBalanceData(address, 0, 1)); - List dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, 0); + List dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, 0, new ArrayList<>(0)); Assert.assertNotNull(dynamics); Assert.assertEquals(1, dynamics.size()); @@ -145,7 +163,7 @@ public class BalanceRecorderUtilsTests { List priorBalances = new ArrayList<>(0); - List dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, 0); + List dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, 0, new ArrayList<>(0)); Assert.assertNotNull(dynamics); Assert.assertEquals(1, dynamics.size()); @@ -156,6 +174,55 @@ public class BalanceRecorderUtilsTests { Assert.assertEquals(2, addressAmountData.getAmount()); } + @Test + public void testBuildBalanceDynamicOneAccountAdjustment() { + List balances = new ArrayList<>(1); + balances.add(new AccountBalanceData(RECIPIENT_ADDRESS, 0, 20)); + + List priorBalances = new ArrayList<>(0); + priorBalances.add(new AccountBalanceData(RECIPIENT_ADDRESS, 0, 12)); + + List transactions = new ArrayList<>(); + + final long amount = 5L; + final long fee = 1L; + + boolean exceptionThrown = false; + + try { + byte[] creatorPublicKey = TestUtils.generatePublicKey(); + + PaymentTransactionData paymentData + = new PaymentTransactionData( + new BaseTransactionData(0L, 0, null, creatorPublicKey, fee, null), + RECIPIENT_ADDRESS, + amount + ); + + transactions.add(paymentData); + + List dynamics + = BalanceRecorderUtils.buildBalanceDynamics( + balances, + priorBalances, + 0, + transactions + ); + + Assert.assertNotNull(dynamics); + Assert.assertEquals(1, dynamics.size()); + + AddressAmountData addressAmountData = dynamics.get(0); + Assert.assertNotNull(addressAmountData); + Assert.assertEquals(RECIPIENT_ADDRESS, addressAmountData.getAddress()); + Assert.assertEquals(3, addressAmountData.getAmount()); + } catch( Exception e ) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + @Test public void testBuildBalanceDynamicsTwoAccountsNegativeValues() { @@ -170,7 +237,7 @@ public class BalanceRecorderUtilsTests { priorBalances.add(new AccountBalanceData(address2, 0, 200)); priorBalances.add(new AccountBalanceData(address1, 0, 5000)); - List dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, -100L); + List dynamics = BalanceRecorderUtils.buildBalanceDynamics(balances, priorBalances, -100L, new ArrayList<>(0)); Assert.assertNotNull(dynamics); Assert.assertEquals(2, dynamics.size()); @@ -304,10 +371,10 @@ public class BalanceRecorderUtilsTests { CopyOnWriteArrayList dynamics = new CopyOnWriteArrayList<>(); - BlockHeightRange range1 = new BlockHeightRange(10, 20); + BlockHeightRange range1 = new BlockHeightRange(10, 20, false); dynamics.add(new BlockHeightRangeAddressAmounts(range1, new ArrayList<>())); - BlockHeightRange range2 = new BlockHeightRange(1, 4); + BlockHeightRange range2 = new BlockHeightRange(1, 4, false); dynamics.add(new BlockHeightRangeAddressAmounts(range2, new ArrayList<>())); Assert.assertEquals(2, dynamics.size()); @@ -323,13 +390,13 @@ public class BalanceRecorderUtilsTests { CopyOnWriteArrayList dynamics = new CopyOnWriteArrayList<>(); - BlockHeightRange range1 = new BlockHeightRange(1,5); + BlockHeightRange range1 = new BlockHeightRange(1,5, false); dynamics.add(new BlockHeightRangeAddressAmounts(range1, new ArrayList<>())); - BlockHeightRange range2 = new BlockHeightRange(6, 11); + BlockHeightRange range2 = new BlockHeightRange(6, 11, false); dynamics.add((new BlockHeightRangeAddressAmounts(range2, new ArrayList<>()))); - BlockHeightRange range3 = new BlockHeightRange(22, 16); + BlockHeightRange range3 = new BlockHeightRange(22, 16, false); dynamics.add(new BlockHeightRangeAddressAmounts(range3, new ArrayList<>())); Assert.assertEquals(3, dynamics.size()); @@ -344,18 +411,353 @@ public class BalanceRecorderUtilsTests { public void testRemoveOldestDynamicsTwice() { CopyOnWriteArrayList dynamics = new CopyOnWriteArrayList<>(); - dynamics.add(new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 5), new ArrayList<>())); - dynamics.add(new BlockHeightRangeAddressAmounts(new BlockHeightRange(5, 9), new ArrayList<>())); + dynamics.add(new BlockHeightRangeAddressAmounts(new BlockHeightRange(1, 5, false), new ArrayList<>())); + dynamics.add(new BlockHeightRangeAddressAmounts(new BlockHeightRange(5, 9, false), new ArrayList<>())); Assert.assertEquals(2, dynamics.size()); BalanceRecorderUtils.removeOldestDynamics(dynamics); Assert.assertEquals(1, dynamics.size()); - Assert.assertTrue(dynamics.get(0).getRange().equals(new BlockHeightRange(5, 9))); + Assert.assertTrue(dynamics.get(0).getRange().equals(new BlockHeightRange(5, 9, false))); BalanceRecorderUtils.removeOldestDynamics(dynamics); Assert.assertEquals(0, dynamics.size()); } + + @Test + public void testMapBalanceModificationsForPaymentTransaction() { + + boolean exceptionThrown = false; + + try { + final long amount = 1L; + final long fee = 1L; + + byte[] creatorPublicKey = TestUtils.generatePublicKey(); + + PaymentTransactionData paymentData + = new PaymentTransactionData( + new BaseTransactionData(0L, 0, null, creatorPublicKey, fee, null), + RECIPIENT_ADDRESS, + amount + ); + + // map balance modifications for addresses in the transaction + Map amountsByAddress = new HashMap<>(); + BalanceRecorderUtils.mapBalanceModicationsForPaymentTransaction(amountsByAddress, paymentData); + + // this will not add the fee, that is done in a different place + assertAmountsByAddress(amountsByAddress, amount, creatorPublicKey, RECIPIENT_ADDRESS); + } catch (Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testMapBalanceModificationsForAssetOrderTransaction() { + + boolean exceptionThrown = false; + + try{ + final long amount = 1L; + final long fee = 1L; + + byte[] creatorPublicKey = TestUtils.generatePublicKey(); + + TransferAssetTransactionData transferAssetData + = new TransferAssetTransactionData( + new BaseTransactionData(0L, 0, null, creatorPublicKey, fee, null), + RECIPIENT_ADDRESS, + amount, + 0 + ); + + // map balance modifications for addresses in the transaction + Map amountsByAddress = new HashMap<>(); + BalanceRecorderUtils.mapBalanceModificationsForTransferAssetTransaction(amountsByAddress, transferAssetData); + + assertAmountsByAddress(amountsByAddress, amount, creatorPublicKey, RECIPIENT_ADDRESS); + } catch( Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testMapBalanceModificationsForATTransactionMessageType() { + + boolean exceptionThrown = false; + + try { + + final long fee = 1L; + + byte[] creatorPublicKey = TestUtils.generatePublicKey(); + Map amountsByAddress = new HashMap<>(); + + ATTransactionData atTransactionData = new ATTransactionData(new BaseTransactionData(0L, 0, null, creatorPublicKey, fee, null), + AT_ADDRESS, + RECIPIENT_ADDRESS, + new byte[0]); + BalanceRecorderUtils.mapBalanceModificationsForAtTransaction( amountsByAddress, atTransactionData); + + // no balance changes for AT message + Assert.assertTrue(amountsByAddress.size() == 0); + } catch( Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testMapBalanceModificationsForATTransactionPaymentType() { + + boolean exceptionThrown = false; + + try{ + final long amount = 1L; + final long fee = 1L; + + byte[] creatorPublicKey = TestUtils.generatePublicKey(); + + Map amountsByAddress = new HashMap<>(); + + ATTransactionData atTransactionData + = new ATTransactionData( + new BaseTransactionData(0L, 0, null, creatorPublicKey, fee, null), + AT_ADDRESS, + RECIPIENT_ADDRESS, + amount, + 0 + ); + + BalanceRecorderUtils.mapBalanceModificationsForAtTransaction( amountsByAddress, atTransactionData); + + assertAmountByAddress(amountsByAddress, amount, RECIPIENT_ADDRESS); + + assertAmountByAddress(amountsByAddress, -amount, AT_ADDRESS); + } catch( Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testMapBalanceModificationsForBuyNameTransaction() { + + boolean exceptionThrown = false; + + try{ + final long amount = 100L; + final long fee = 1L; + + byte[] creatorPublicKey = TestUtils.generatePublicKey(); + Map amountsByAddress = new HashMap<>(); + + BuyNameTransactionData buyNameData + = new BuyNameTransactionData( + new BaseTransactionData(0L, 0, null, creatorPublicKey, fee, null), + "null", + amount, + RECIPIENT_ADDRESS + ); + + BalanceRecorderUtils.mapBalanceModificationsForBuyNameTransaction(amountsByAddress, buyNameData); + + assertAmountsByAddress(amountsByAddress, amount, creatorPublicKey, RECIPIENT_ADDRESS); + } catch( Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testMapBalanceModificationsForMultiPaymentTransaction() { + + boolean exceptionThrown = false; + + try{ + final long amount = 100L; + final long fee = 1L; + + byte[] creatorPublicKey = TestUtils.generatePublicKey(); + Map amountsByAddress = new HashMap<>(); + + List payments = new ArrayList<>(); + + payments.add(new PaymentData(RECIPIENT_ADDRESS, 0, amount)); + + MultiPaymentTransactionData multiPayment + = new MultiPaymentTransactionData(new BaseTransactionData(0L, 0, null, creatorPublicKey, fee, null), + payments); + BalanceRecorderUtils.mapBalanceModificationsForMultiPaymentTransaction(amountsByAddress,multiPayment); + assertAmountsByAddress(amountsByAddress, amount, creatorPublicKey, RECIPIENT_ADDRESS); + } catch( Exception e ) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testMapBalanceModificationsForMultiPaymentTransaction2PaymentsOneAddress() { + + boolean exceptionThrown = false; + + try{ + final long amount = 100L; + final long fee = 1L; + + byte[] creatorPublicKey = TestUtils.generatePublicKey(); + Map amountsByAddress = new HashMap<>(); + + List payments = new ArrayList<>(); + + payments.add(new PaymentData(RECIPIENT_ADDRESS, 0, amount)); + payments.add(new PaymentData(RECIPIENT_ADDRESS, 0, amount)); + + MultiPaymentTransactionData multiPayment + = new MultiPaymentTransactionData(new BaseTransactionData(0L, 0, null, creatorPublicKey, fee, null), + payments); + BalanceRecorderUtils.mapBalanceModificationsForMultiPaymentTransaction(amountsByAddress,multiPayment); + assertAmountsByAddress(amountsByAddress, 2*amount, creatorPublicKey, RECIPIENT_ADDRESS); + } catch( Exception e ) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testMapBalanceModificationsForMultiPaymentTransaction2PaymentsTwoAddresses() { + + boolean exceptionThrown = false; + + try{ + final long amount = 100L; + final long fee = 1L; + + byte[] creatorPublicKey = TestUtils.generatePublicKey(); + Map amountsByAddress = new HashMap<>(); + + List payments = new ArrayList<>(); + + payments.add(new PaymentData(RECIPIENT_ADDRESS, 0, amount)); + payments.add(new PaymentData(OTHER, 0, amount)); + + MultiPaymentTransactionData multiPayment + = new MultiPaymentTransactionData(new BaseTransactionData(0L, 0, null, creatorPublicKey, fee, null), + payments); + BalanceRecorderUtils.mapBalanceModificationsForMultiPaymentTransaction(amountsByAddress,multiPayment); + assertAmountByAddress(amountsByAddress, amount, RECIPIENT_ADDRESS); + assertAmountByAddress(amountsByAddress, amount, OTHER); + + String creatorAddress = Crypto.toAddress(creatorPublicKey); + + assertAmountByAddress(amountsByAddress, 2*-amount, creatorAddress); + } catch( Exception e ) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testMapBalanceModificationsForDeployAtTransaction() { + + boolean exceptionThrown = false; + + try{ + final long amount = 3L; + final long fee = 1L; + + byte[] creatorPublicKey = TestUtils.generatePublicKey(); + Map amountsByAddress = new HashMap<>(); + + DeployAtTransactionData deployAt + = new DeployAtTransactionData( + new BaseTransactionData(0L, 0, null, creatorPublicKey, fee, null), + AT_ADDRESS, "name", "description", "type", "tags", new byte[0], amount, Asset.QORT + ); + + BalanceRecorderUtils.mapBalanceModificationsForDeployAtTransaction(amountsByAddress,deployAt); + assertAmountsByAddress(amountsByAddress, amount, creatorPublicKey, AT_ADDRESS); + } catch( Exception e) { + exceptionThrown = true; + e.printStackTrace(); + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testMapBalanceModificationsForTransaction() { + + boolean exceptionThrown = false; + + try { + final long fee = 2; + + byte[] creatorPublicKey = TestUtils.generatePublicKey(); + Map amountsByAddress = new HashMap<>(); + + BalanceRecorderUtils.mapBalanceModificationsForTransaction( + amountsByAddress, + new RegisterNameTransactionData( + new BaseTransactionData(0L, 0, null, creatorPublicKey, fee, null), + "aaa", "data", "aaa") + ); + + String creatorAddress = Crypto.toAddress(creatorPublicKey); + + assertAmountByAddress(amountsByAddress, -fee, creatorAddress); + } catch(Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testBlockHeightRangeEqualityTrue() { + + BlockHeightRange range1 = new BlockHeightRange(2, 4, false); + BlockHeightRange range2 = new BlockHeightRange(2, 4, true); + + Assert.assertTrue(range1.equals(range2)); + Assert.assertEquals(range1, range2); + } + + @Test + public void testBloHeightRangeEqualityFalse() { + + BlockHeightRange range1 = new BlockHeightRange(2, 3, true); + BlockHeightRange range2 = new BlockHeightRange(2, 4, true); + + Assert.assertFalse(range1.equals(range2)); + } + + private static void assertAmountsByAddress(Map amountsByAddress, long amount, byte[] creatorPublicKey, String recipientAddress) { + assertAmountByAddress(amountsByAddress, amount, recipientAddress); + + String creatorAddress = Crypto.toAddress(creatorPublicKey); + + assertAmountByAddress(amountsByAddress, -amount, creatorAddress); + } + + private static void assertAmountByAddress(Map amountsByAddress, long amount, String address) { + Long amountForAddress = amountsByAddress.get(address); + + Assert.assertTrue(amountsByAddress.containsKey(address)); + Assert.assertNotNull(amountForAddress); + Assert.assertEquals(amount, amountForAddress.longValue()); + } } \ No newline at end of file diff --git a/src/test/java/org/qortal/test/utils/TestUtils.java b/src/test/java/org/qortal/test/utils/TestUtils.java new file mode 100644 index 00000000..b66591a0 --- /dev/null +++ b/src/test/java/org/qortal/test/utils/TestUtils.java @@ -0,0 +1,48 @@ +package org.qortal.test.utils; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; + +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.MessageDigest; +import java.security.PublicKey; +import java.security.Security; + +public class TestUtils { + public static byte[] generatePublicKey() throws Exception { + // Add the Bouncy Castle provider + Security.addProvider(new BouncyCastleProvider()); + + // Generate a key pair + KeyPair keyPair = generateKeyPair(); + + // Get the public key + PublicKey publicKey = keyPair.getPublic(); + + // Get the public key as a byte array + byte[] publicKeyBytes = publicKey.getEncoded(); + + // Generate a RIPEMD160 message digest from the public key + byte[] ripeMd160Digest = generateRipeMd160Digest(publicKeyBytes); + + return ripeMd160Digest; + } + + public static KeyPair generateKeyPair() throws Exception { + // Generate a key pair using the RSA algorithm + KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); + keyGen.initialize(2048); // Key size (bits) + return keyGen.generateKeyPair(); + } + + public static byte[] generateRipeMd160Digest(byte[] input) throws Exception { + // Create a RIPEMD160 message digest instance + MessageDigest ripeMd160 = MessageDigest.getInstance("RIPEMD160", new BouncyCastleProvider()); + + // Update the message digest with the input bytes + ripeMd160.update(input); + + // Get the message digest bytes + return ripeMd160.digest(); + } +} From 3952705eddde8baa6c5ef878b4aa095629a692ec Mon Sep 17 00:00:00 2001 From: kennycud Date: Thu, 26 Dec 2024 13:53:00 -0800 Subject: [PATCH 07/24] Admin replace founders hardfork and online validation fail-safe hardfork. --- src/main/java/org/qortal/block/Block.java | 165 +++++++++++++++--- .../java/org/qortal/block/BlockChain.java | 12 +- src/main/resources/blockchain.json | 4 +- 3 files changed, 154 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index c9353d70..a296fd0a 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -23,6 +23,7 @@ import org.qortal.data.at.ATStateData; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.block.BlockTransactionData; +import org.qortal.data.group.GroupAdminData; import org.qortal.data.network.OnlineAccountData; import org.qortal.data.transaction.TransactionData; import org.qortal.repository.*; @@ -170,6 +171,19 @@ public class Block { } } + /** + * Get Effective Minting Level + * + * @return the effective minting level, if a data exception is thrown, it catches the exception and returns a zero + */ + public int getEffectiveMintingLevel() { + try { + return this.mintingAccount.getEffectiveMintingLevel(); + } catch (DataException e) { + return 0; + } + } + public Account getMintingAccount() { return this.mintingAccount; } @@ -186,7 +200,7 @@ public class Block { * @return account-level share "bin" from blockchain config, or null if founder / none found */ public AccountLevelShareBin getShareBin(int blockHeight) { - if (this.isMinterFounder) + if (this.isMinterFounder && blockHeight < BlockChain.getInstance().getAdminsReplaceFoundersHeight()) return null; final int accountLevel = this.mintingAccountData.getLevel(); @@ -736,15 +750,7 @@ public class Block { List expandedAccounts = new ArrayList<>(); for (RewardShareData rewardShare : this.cachedOnlineRewardShares) { - int groupId = BlockChain.getInstance().getMintingGroupId(); - String address = rewardShare.getMinter(); - boolean isMinterGroupMember = repository.getGroupRepository().memberExists(groupId, address); - - if (this.getBlockData().getHeight() < BlockChain.getInstance().getFixBatchRewardHeight()) - expandedAccounts.add(new ExpandedAccount(repository, rewardShare)); - - if (this.getBlockData().getHeight() >= BlockChain.getInstance().getFixBatchRewardHeight() && isMinterGroupMember) - expandedAccounts.add(new ExpandedAccount(repository, rewardShare)); + expandedAccounts.add(new ExpandedAccount(repository, rewardShare)); } this.cachedExpandedAccounts = expandedAccounts; @@ -1156,20 +1162,32 @@ public class Block { // After feature trigger, require all online account minters to be greater than level 0 if (this.getBlockData().getHeight() >= BlockChain.getInstance().getOnlineAccountMinterLevelValidationHeight()) { - List expandedAccounts = this.getExpandedAccounts(); - for (ExpandedAccount account : expandedAccounts) { - int groupId = BlockChain.getInstance().getMintingGroupId(); - String address = account.getMintingAccount().getAddress(); - boolean isMinterGroupMember = repository.getGroupRepository().memberExists(groupId, address); + if( this.blockData.getHeight() < BlockChain.getInstance().getOnlineValidationFailSafeHeight()) { + List expandedAccounts + = this.getExpandedAccounts().stream() + .filter(expandedAccount -> expandedAccount.isMinterMember) + .collect(Collectors.toList()); - if (account.getMintingAccount().getEffectiveMintingLevel() == 0) - return ValidationResult.ONLINE_ACCOUNTS_INVALID; - - if (this.getBlockData().getHeight() >= BlockChain.getInstance().getFixBatchRewardHeight()) { - if (!isMinterGroupMember) + for (ExpandedAccount account : expandedAccounts) { + if (account.getMintingAccount().getEffectiveMintingLevel() == 0) return ValidationResult.ONLINE_ACCOUNTS_INVALID; + + if (this.getBlockData().getHeight() >= BlockChain.getInstance().getFixBatchRewardHeight()) { + if (!account.isMinterMember) + return ValidationResult.ONLINE_ACCOUNTS_INVALID; + } } } + // this.blockData.getHeight() >= BlockChain.getInstance().getOnlineValidationFailSafeHeight() + else { + Optional anyInvalidAccount + = this.getExpandedAccounts().stream() + .filter( + account -> account.getEffectiveMintingLevel() == 0 || + !account.isMinterMember) + .findAny(); + if( anyInvalidAccount.isPresent() ) return ValidationResult.ONLINE_ACCOUNTS_INVALID; + } } // If block is past a certain age then we simply assume the signatures were correct @@ -1659,7 +1677,17 @@ public class Block { final List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); final int maximumLevel = cumulativeBlocksByLevel.size() - 1; - final List expandedAccounts = this.getExpandedAccounts(); + final List expandedAccounts; + + if (this.getBlockData().getHeight() < BlockChain.getInstance().getFixBatchRewardHeight()) { + expandedAccounts = this.getExpandedAccounts().stream().collect(Collectors.toList()); + } + else { + expandedAccounts + = this.getExpandedAccounts().stream() + .filter(expandedAccount -> expandedAccount.isMinterMember) + .collect(Collectors.toList()); + } Set allUniqueExpandedAccounts = new HashSet<>(); for (ExpandedAccount expandedAccount : expandedAccounts) { @@ -2059,7 +2087,17 @@ public class Block { final List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); final int maximumLevel = cumulativeBlocksByLevel.size() - 1; - final List expandedAccounts = this.getExpandedAccounts(); + final List expandedAccounts; + + if (this.getBlockData().getHeight() < BlockChain.getInstance().getFixBatchRewardHeight()) { + expandedAccounts = this.getExpandedAccounts().stream().collect(Collectors.toList()); + } + else { + expandedAccounts + = this.getExpandedAccounts().stream() + .filter(expandedAccount -> expandedAccount.isMinterMember) + .collect(Collectors.toList()); + } Set allUniqueExpandedAccounts = new HashSet<>(); for (ExpandedAccount expandedAccount : expandedAccounts) { @@ -2263,7 +2301,17 @@ public class Block { List rewardCandidates = new ArrayList<>(); // All online accounts - final List expandedAccounts = this.getExpandedAccounts(); + final List expandedAccounts; + + if (this.getBlockData().getHeight() < BlockChain.getInstance().getFixBatchRewardHeight()) { + expandedAccounts = this.getExpandedAccounts().stream().collect(Collectors.toList()); + } + else { + expandedAccounts + = this.getExpandedAccounts().stream() + .filter(expandedAccount -> expandedAccount.isMinterMember) + .collect(Collectors.toList()); + } /* * Distribution rules: @@ -2388,7 +2436,7 @@ public class Block { final long qoraHoldersShare = BlockChain.getInstance().getQoraHoldersShareAtHeight(this.blockData.getHeight()); // Perform account-level-based reward scaling if appropriate - if (!haveFounders) { + if (!haveFounders && this.blockData.getHeight() < BlockChain.getInstance().getAdminsReplaceFoundersHeight() ) { // Recalculate distribution ratios based on candidates // Nothing shared? This shouldn't happen @@ -2424,7 +2472,7 @@ public class Block { } // Add founders as reward candidate if appropriate - if (haveFounders) { + if (haveFounders && this.blockData.getHeight() < BlockChain.getInstance().getAdminsReplaceFoundersHeight()) { // Yes: add to reward candidates list BlockRewardDistributor founderDistributor = (distributionAmount, balanceChanges) -> distributeBlockRewardShare(distributionAmount, onlineFounderAccounts, balanceChanges); @@ -2432,10 +2480,77 @@ public class Block { BlockRewardCandidate rewardCandidate = new BlockRewardCandidate("Founders", foundersShare, founderDistributor); rewardCandidates.add(rewardCandidate); } + else if (this.blockData.getHeight() >= BlockChain.getInstance().getAdminsReplaceFoundersHeight()) { + try (final Repository repository = RepositoryManager.getRepository()) { + GroupRepository groupRepository = repository.getGroupRepository(); + + // all minter admins + List minterAdmins + = groupRepository.getGroupAdmins(BlockChain.getInstance().getMintingGroupId()).stream() + .map(GroupAdminData::getAdmin) + .collect(Collectors.toList()); + + // all minter admins that are online + List onlineMinterAdminAccounts + = expandedAccounts.stream() + .filter(expandedAccount -> minterAdmins.contains(expandedAccount.getMintingAccount().getAddress())) + .collect(Collectors.toList()); + + BlockRewardDistributor minterAdminDistributor + = (distributionAmount, balanceChanges) + -> + distributeBlockRewardShare(distributionAmount, onlineMinterAdminAccounts, balanceChanges); + + long adminShare = 1_00000000 - totalShares; + + long minterAdminShare = adminShare / 2; + BlockRewardCandidate minterAdminRewardCandidate + = new BlockRewardCandidate("Minter Admins", minterAdminShare, minterAdminDistributor); + rewardCandidates.add(minterAdminRewardCandidate); + + totalShares -= adminShare; + + // all dev admins + List devAdminAddresses + = groupRepository.getGroupAdmins(1).stream() + .map(GroupAdminData::getAdmin) + .collect(Collectors.toList()); + + BlockRewardDistributor devAdminDistributor + = (distributionAmount, balanceChanges) -> distributeToAccounts(distributionAmount, devAdminAddresses, balanceChanges); + + long devAdminShare = 1_00000000 - totalShares; + BlockRewardCandidate devAdminRewardCandidate + = new BlockRewardCandidate("Dev Admins", devAdminShare,devAdminDistributor); + rewardCandidates.add(devAdminRewardCandidate); + } + } return rewardCandidates; } + /** + * Distribute To Accounts + * + * Merges distribute shares to a map of distribution shares. + * + * @param distributionAmount the amount to distribute + * @param accountAddressess the addresses to distribute to + * @param balanceChanges the map of distribution shares, this gets appended to + * + * @return the total amount mapped to addresses for distribution + */ + public static long distributeToAccounts(long distributionAmount, List accountAddressess, Map balanceChanges) { + + long distibutionShare = distributionAmount / accountAddressess.size(); + + for(String accountAddress : accountAddressess ) { + balanceChanges.merge(accountAddress, distibutionShare, Long::sum); + } + + return distibutionShare * accountAddressess.size(); + } + private static long distributeBlockRewardShare(long distributionAmount, List accounts, Map balanceChanges) { // Collate all expanded accounts by minting account Map> accountsByMinter = new HashMap<>(); diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 3e98d4a0..378d6fb5 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -88,7 +88,9 @@ public class BlockChain { onlyMintWithNameHeight, removeOnlyMintWithNameHeight, groupMemberCheckHeight, - fixBatchRewardHeight + fixBatchRewardHeight, + adminsReplaceFoundersHeight, + onlineValidationFailSafeHeight } // Custom transaction fees @@ -662,6 +664,14 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.fixBatchRewardHeight.name()).intValue(); } + public int getAdminsReplaceFoundersHeight() { + return this.featureTriggers.get(FeatureTrigger.adminsReplaceFoundersHeight.name()).intValue(); + } + + public int getOnlineValidationFailSafeHeight() { + return this.featureTriggers.get(FeatureTrigger.onlineValidationFailSafeHeight.name()).intValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 762a5708..70622061 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -113,7 +113,9 @@ "onlyMintWithNameHeight": 1900300, "removeOnlyMintWithNameHeight": 1935500, "groupMemberCheckHeight": 1902700, - "fixBatchRewardHeight": 1945900 + "fixBatchRewardHeight": 1945900, + "adminsReplaceFoundersHeight": 9999999, + "onlineValidationFailSafeHeight": 9999999 }, "checkpoints": [ { "height": 1136300, "signature": "3BbwawEF2uN8Ni5ofpJXkukoU8ctAPxYoFB7whq9pKfBnjfZcpfEJT4R95NvBDoTP8WDyWvsUvbfHbcr9qSZuYpSKZjUQTvdFf6eqznHGEwhZApWfvXu6zjGCxYCp65F4jsVYYJjkzbjmkCg5WAwN5voudngA23kMK6PpTNygapCzXt" } From 45d88c1bac68262ac197d120afb95330d310e835 Mon Sep 17 00:00:00 2001 From: kennycud Date: Thu, 26 Dec 2024 14:40:44 -0800 Subject: [PATCH 08/24] Admin share typo fix and new test case submission. --- src/main/java/org/qortal/block/Block.java | 2 +- .../org/qortal/test/block/BlockTests.java | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/qortal/test/block/BlockTests.java diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index a296fd0a..35e217c1 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -2508,7 +2508,7 @@ public class Block { = new BlockRewardCandidate("Minter Admins", minterAdminShare, minterAdminDistributor); rewardCandidates.add(minterAdminRewardCandidate); - totalShares -= adminShare; + totalShares -= minterAdminShare; // all dev admins List devAdminAddresses diff --git a/src/test/java/org/qortal/test/block/BlockTests.java b/src/test/java/org/qortal/test/block/BlockTests.java new file mode 100644 index 00000000..cd6f37e8 --- /dev/null +++ b/src/test/java/org/qortal/test/block/BlockTests.java @@ -0,0 +1,58 @@ +package org.qortal.test.block; + +import org.checkerframework.checker.units.qual.K; +import org.junit.Assert; +import org.junit.Test; +import org.qortal.block.Block; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; + +public class BlockTests { + + @Test + public void testDistributeToAccountsOneDistribution(){ + List addresses = new ArrayList<>(); + addresses.add("a"); + addresses.add("b"); + addresses.add("c"); + + HashMap balanceByAddress = new HashMap<>(); + long total = Block.distributeToAccounts( 10L, addresses, balanceByAddress); + + Assert.assertEquals(9, total); + + Assert.assertEquals(3, balanceByAddress.size()); + Assert.assertTrue(balanceByAddress.containsKey("a")); + Assert.assertTrue(balanceByAddress.containsKey("b")); + Assert.assertTrue(balanceByAddress.containsKey("c")); + Assert.assertEquals(3L, balanceByAddress.getOrDefault("a", 0L).longValue()); + Assert.assertEquals(3L, balanceByAddress.getOrDefault("b", 0L).longValue()); + Assert.assertEquals(3L, balanceByAddress.getOrDefault("c", 0L).longValue()); + } + + @Test + public void testDistributeToAccountsTwoDistributions(){ + List addresses = new ArrayList<>(); + addresses.add("a"); + addresses.add("b"); + addresses.add("c"); + + HashMap balanceByAddress = new HashMap<>(); + long total1 = Block.distributeToAccounts( 10L, addresses, balanceByAddress); + long total2 = Block.distributeToAccounts( 20L, addresses, balanceByAddress); + + Assert.assertEquals(9, total1); + Assert.assertEquals(18, total2); + + Assert.assertEquals(3, balanceByAddress.size()); + Assert.assertTrue(balanceByAddress.containsKey("a")); + Assert.assertTrue(balanceByAddress.containsKey("b")); + Assert.assertTrue(balanceByAddress.containsKey("c")); + Assert.assertEquals(9L, balanceByAddress.getOrDefault("a", 0L).longValue()); + Assert.assertEquals(9L, balanceByAddress.getOrDefault("b", 0L).longValue()); + Assert.assertEquals(9L, balanceByAddress.getOrDefault("c", 0L).longValue()); + } +} From bdbbd0152fbbcf5e5428fca7e667c9e4da37ae75 Mon Sep 17 00:00:00 2001 From: kennycud Date: Sat, 28 Dec 2024 14:01:01 -0800 Subject: [PATCH 09/24] updated the hard fork heights for the test chain --- src/test/resources/test-chain-v2.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index c829975b..689b900b 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -103,7 +103,10 @@ "onlyMintWithNameHeight": 9999999999990, "groupMemberCheckHeight": 9999999999999, "decreaseOnlineAccountsDifficultyTimestamp": 9999999999999, - "removeOnlyMintWithNameHeight": 9999999999999 + "removeOnlyMintWithNameHeight": 9999999999999, + "fixBatchRewardHeight": 9999999999999, + "adminsReplaceFoundersHeight": 9999999999999, + "onlineValidationFailSafeHeight": 9999999999999 }, "genesisInfo": { "version": 4, From a300ac2393bc0823e2a815353606a4bab497a59e Mon Sep 17 00:00:00 2001 From: kennycud Date: Sun, 29 Dec 2024 18:08:04 -0800 Subject: [PATCH 10/24] added capabilities for groups with null ownership including banning and kicking members and member ban cancellations; enforcing group approval thresholds to invites and invite cancellations; the established add and remove admin capabilities were used as guidance for this implementation; this was added as a hardfork to preserve group transactions from previous blocks --- .../java/org/qortal/block/BlockChain.java | 7 +- .../hsqldb/HSQLDBGroupRepository.java | 26 +- .../CancelGroupBanTransaction.java | 25 +- .../CancelGroupInviteTransaction.java | 12 + .../transaction/GroupBanTransaction.java | 24 +- .../transaction/GroupInviteTransaction.java | 12 + .../transaction/GroupKickTransaction.java | 25 +- .../org/qortal/transaction/Transaction.java | 10 +- src/main/resources/blockchain.json | 3 +- .../qortal/test/group/DevGroupAdminTests.java | 447 +++++++++++++++--- src/test/resources/test-chain-v2.json | 3 +- 11 files changed, 497 insertions(+), 97 deletions(-) diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 378d6fb5..42da0b3a 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -90,7 +90,8 @@ public class BlockChain { groupMemberCheckHeight, fixBatchRewardHeight, adminsReplaceFoundersHeight, - onlineValidationFailSafeHeight + onlineValidationFailSafeHeight, + nullGroupMembershipHeight } // Custom transaction fees @@ -672,6 +673,10 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.onlineValidationFailSafeHeight.name()).intValue(); } + public int getNullGroupMembershipHeight() { + return this.featureTriggers.get(FeatureTrigger.nullGroupMembershipHeight.name()).intValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java index 9c7521fc..a364ed32 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java @@ -24,7 +24,7 @@ public class HSQLDBGroupRepository implements GroupRepository { public GroupData fromGroupId(int groupId) throws DataException { String sql = "SELECT group_name, owner, description, created_when, updated_when, reference, is_open, " + "approval_threshold, min_block_delay, max_block_delay, creation_group_id, reduced_group_name " - + "FROM Groups WHERE group_id = ?"; + + "FROM `Groups` WHERE group_id = ?"; try (ResultSet resultSet = this.repository.checkedExecute(sql, groupId)) { if (resultSet == null) @@ -62,7 +62,7 @@ public class HSQLDBGroupRepository implements GroupRepository { public GroupData fromGroupName(String groupName) throws DataException { String sql = "SELECT group_id, owner, description, created_when, updated_when, reference, is_open, " + "approval_threshold, min_block_delay, max_block_delay, creation_group_id, reduced_group_name " - + "FROM Groups WHERE group_name = ?"; + + "FROM `Groups` WHERE group_name = ?"; try (ResultSet resultSet = this.repository.checkedExecute(sql, groupName)) { if (resultSet == null) @@ -99,7 +99,7 @@ public class HSQLDBGroupRepository implements GroupRepository { @Override public boolean groupExists(int groupId) throws DataException { try { - return this.repository.exists("Groups", "group_id = ?", groupId); + return this.repository.exists("`Groups`", "group_id = ?", groupId); } catch (SQLException e) { throw new DataException("Unable to check for group in repository", e); } @@ -108,7 +108,7 @@ public class HSQLDBGroupRepository implements GroupRepository { @Override public boolean groupExists(String groupName) throws DataException { try { - return this.repository.exists("Groups", "group_name = ?", groupName); + return this.repository.exists("`Groups`", "group_name = ?", groupName); } catch (SQLException e) { throw new DataException("Unable to check for group in repository", e); } @@ -117,7 +117,7 @@ public class HSQLDBGroupRepository implements GroupRepository { @Override public boolean reducedGroupNameExists(String reducedGroupName) throws DataException { try { - return this.repository.exists("Groups", "reduced_group_name = ?", reducedGroupName); + return this.repository.exists("`Groups`", "reduced_group_name = ?", reducedGroupName); } catch (SQLException e) { throw new DataException("Unable to check for reduced group name in repository", e); } @@ -129,7 +129,7 @@ public class HSQLDBGroupRepository implements GroupRepository { sql.append("SELECT group_id, owner, group_name, description, created_when, updated_when, reference, is_open, " + "approval_threshold, min_block_delay, max_block_delay, creation_group_id, reduced_group_name " - + "FROM Groups ORDER BY group_name"); + + "FROM `Groups` ORDER BY group_name"); if (reverse != null && reverse) sql.append(" DESC"); @@ -181,7 +181,7 @@ public class HSQLDBGroupRepository implements GroupRepository { sql.append("SELECT group_id, group_name, description, created_when, updated_when, reference, is_open, " + "approval_threshold, min_block_delay, max_block_delay, creation_group_id, reduced_group_name " - + "FROM Groups WHERE owner = ? ORDER BY group_name"); + + "FROM `Groups` WHERE owner = ? ORDER BY group_name"); if (reverse != null && reverse) sql.append(" DESC"); @@ -231,7 +231,7 @@ public class HSQLDBGroupRepository implements GroupRepository { StringBuilder sql = new StringBuilder(512); sql.append("SELECT group_id, owner, group_name, description, created_when, updated_when, reference, is_open, " - + "approval_threshold, min_block_delay, max_block_delay, creation_group_id, reduced_group_name, admin FROM Groups " + + "approval_threshold, min_block_delay, max_block_delay, creation_group_id, reduced_group_name, admin FROM `Groups` " + "JOIN GroupMembers USING (group_id) " + "LEFT OUTER JOIN GroupAdmins ON GroupAdmins.group_id = GroupMembers.group_id AND GroupAdmins.admin = GroupMembers.address " + "WHERE address = ? ORDER BY group_name"); @@ -289,7 +289,7 @@ public class HSQLDBGroupRepository implements GroupRepository { @Override public void save(GroupData groupData) throws DataException { - HSQLDBSaver saveHelper = new HSQLDBSaver("Groups"); + HSQLDBSaver saveHelper = new HSQLDBSaver("`Groups`"); saveHelper.bind("group_id", groupData.getGroupId()).bind("owner", groupData.getOwner()).bind("group_name", groupData.getGroupName()) .bind("description", groupData.getDescription()).bind("created_when", groupData.getCreated()).bind("updated_when", groupData.getUpdated()) @@ -302,7 +302,7 @@ public class HSQLDBGroupRepository implements GroupRepository { if (groupData.getGroupId() == null) { // Fetch new groupId - try (ResultSet resultSet = this.repository.checkedExecute("SELECT group_id FROM Groups WHERE reference = ?", groupData.getReference())) { + try (ResultSet resultSet = this.repository.checkedExecute("SELECT group_id FROM `Groups` WHERE reference = ?", groupData.getReference())) { if (resultSet == null) throw new DataException("Unable to fetch new group ID from repository"); @@ -318,7 +318,7 @@ public class HSQLDBGroupRepository implements GroupRepository { public void delete(int groupId) throws DataException { try { // Remove group - this.repository.delete("Groups", "group_id = ?", groupId); + this.repository.delete("`Groups`", "group_id = ?", groupId); } catch (SQLException e) { throw new DataException("Unable to delete group info from repository", e); } @@ -328,7 +328,7 @@ public class HSQLDBGroupRepository implements GroupRepository { public void delete(String groupName) throws DataException { try { // Remove group - this.repository.delete("Groups", "group_name = ?", groupName); + this.repository.delete("`Groups`", "group_name = ?", groupName); } catch (SQLException e) { throw new DataException("Unable to delete group info from repository", e); } @@ -338,7 +338,7 @@ public class HSQLDBGroupRepository implements GroupRepository { @Override public String getOwner(int groupId) throws DataException { - try (ResultSet resultSet = this.repository.checkedExecute("SELECT owner FROM Groups WHERE group_id = ?", groupId)) { + try (ResultSet resultSet = this.repository.checkedExecute("SELECT owner FROM `Groups` WHERE group_id = ?", groupId)) { if (resultSet == null) return null; diff --git a/src/main/java/org/qortal/transaction/CancelGroupBanTransaction.java b/src/main/java/org/qortal/transaction/CancelGroupBanTransaction.java index f3511ded..95a267f3 100644 --- a/src/main/java/org/qortal/transaction/CancelGroupBanTransaction.java +++ b/src/main/java/org/qortal/transaction/CancelGroupBanTransaction.java @@ -2,6 +2,7 @@ package org.qortal.transaction; import org.qortal.account.Account; import org.qortal.asset.Asset; +import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; import org.qortal.data.group.GroupData; import org.qortal.data.transaction.CancelGroupBanTransactionData; @@ -12,6 +13,7 @@ import org.qortal.repository.Repository; import java.util.Collections; import java.util.List; +import java.util.Objects; public class CancelGroupBanTransaction extends Transaction { @@ -70,9 +72,26 @@ public class CancelGroupBanTransaction extends Transaction { if (!this.repository.getGroupRepository().adminExists(groupId, admin.getAddress())) return ValidationResult.NOT_GROUP_ADMIN; - // Can't unban if not group's current owner - if (!admin.getAddress().equals(groupData.getOwner())) - return ValidationResult.INVALID_GROUP_OWNER; + if( this.repository.getBlockRepository().getBlockchainHeight() < BlockChain.getInstance().getNullGroupMembershipHeight() ) { + // Can't cancel ban if not group's current owner + if (!admin.getAddress().equals(groupData.getOwner())) + return ValidationResult.INVALID_GROUP_OWNER; + } + // if( this.repository.getBlockRepository().getBlockchainHeight() >= BlockChain.getInstance().getNullGroupMembershipHeight() ) + else { + String groupOwner = this.repository.getGroupRepository().getOwner(groupId); + boolean groupOwnedByNullAccount = Objects.equals(groupOwner, Group.NULL_OWNER_ADDRESS); + + // if null ownership group, then check for admin approval + if(groupOwnedByNullAccount ) { + // Require approval if transaction relates to a group owned by the null account + if (!this.needsGroupApproval()) + return ValidationResult.GROUP_APPROVAL_REQUIRED; + } + // Can't cancel ban if not group's current owner + else if (!admin.getAddress().equals(groupData.getOwner())) + return ValidationResult.INVALID_GROUP_OWNER; + } Account member = getMember(); diff --git a/src/main/java/org/qortal/transaction/CancelGroupInviteTransaction.java b/src/main/java/org/qortal/transaction/CancelGroupInviteTransaction.java index d4306bbe..678aa411 100644 --- a/src/main/java/org/qortal/transaction/CancelGroupInviteTransaction.java +++ b/src/main/java/org/qortal/transaction/CancelGroupInviteTransaction.java @@ -2,6 +2,7 @@ package org.qortal.transaction; import org.qortal.account.Account; import org.qortal.asset.Asset; +import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; import org.qortal.data.group.GroupData; import org.qortal.data.transaction.CancelGroupInviteTransactionData; @@ -12,6 +13,7 @@ import org.qortal.repository.Repository; import java.util.Collections; import java.util.List; +import java.util.Objects; public class CancelGroupInviteTransaction extends Transaction { @@ -80,6 +82,16 @@ public class CancelGroupInviteTransaction extends Transaction { if (admin.getConfirmedBalance(Asset.QORT) < this.cancelGroupInviteTransactionData.getFee()) return ValidationResult.NO_BALANCE; + // if null ownership group, then check for admin approval + if( this.repository.getBlockRepository().getBlockchainHeight() >= BlockChain.getInstance().getNullGroupMembershipHeight() ) { + String groupOwner = this.repository.getGroupRepository().getOwner(groupId); + boolean groupOwnedByNullAccount = Objects.equals(groupOwner, Group.NULL_OWNER_ADDRESS); + + // Require approval if transaction relates to a group owned by the null account + if (groupOwnedByNullAccount && !this.needsGroupApproval()) + return ValidationResult.GROUP_APPROVAL_REQUIRED; + } + return ValidationResult.OK; } diff --git a/src/main/java/org/qortal/transaction/GroupBanTransaction.java b/src/main/java/org/qortal/transaction/GroupBanTransaction.java index 1716d206..143a66fb 100644 --- a/src/main/java/org/qortal/transaction/GroupBanTransaction.java +++ b/src/main/java/org/qortal/transaction/GroupBanTransaction.java @@ -2,6 +2,7 @@ package org.qortal.transaction; import org.qortal.account.Account; import org.qortal.asset.Asset; +import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; import org.qortal.data.group.GroupData; import org.qortal.data.transaction.GroupBanTransactionData; @@ -12,6 +13,7 @@ import org.qortal.repository.Repository; import java.util.Collections; import java.util.List; +import java.util.Objects; public class GroupBanTransaction extends Transaction { @@ -70,9 +72,25 @@ public class GroupBanTransaction extends Transaction { if (!this.repository.getGroupRepository().adminExists(groupId, admin.getAddress())) return ValidationResult.NOT_GROUP_ADMIN; - // Can't ban if not group's current owner - if (!admin.getAddress().equals(groupData.getOwner())) - return ValidationResult.INVALID_GROUP_OWNER; + if( this.repository.getBlockRepository().getBlockchainHeight() < BlockChain.getInstance().getNullGroupMembershipHeight() ) { + // Can't ban if not group's current owner + if (!admin.getAddress().equals(groupData.getOwner())) + return ValidationResult.INVALID_GROUP_OWNER; + } + // if( this.repository.getBlockRepository().getBlockchainHeight() >= BlockChain.getInstance().getNullGroupMembershipHeight() ) + else { + String groupOwner = this.repository.getGroupRepository().getOwner(groupId); + boolean groupOwnedByNullAccount = Objects.equals(groupOwner, Group.NULL_OWNER_ADDRESS); + + // if null ownership group, then check for admin approval + if(groupOwnedByNullAccount ) { + // Require approval if transaction relates to a group owned by the null account + if (!this.needsGroupApproval()) + return ValidationResult.GROUP_APPROVAL_REQUIRED; + } + else if (!admin.getAddress().equals(groupData.getOwner())) + return ValidationResult.INVALID_GROUP_OWNER; + } Account offender = getOffender(); diff --git a/src/main/java/org/qortal/transaction/GroupInviteTransaction.java b/src/main/java/org/qortal/transaction/GroupInviteTransaction.java index e58d1b9c..96179d1b 100644 --- a/src/main/java/org/qortal/transaction/GroupInviteTransaction.java +++ b/src/main/java/org/qortal/transaction/GroupInviteTransaction.java @@ -2,6 +2,7 @@ package org.qortal.transaction; import org.qortal.account.Account; import org.qortal.asset.Asset; +import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; import org.qortal.data.transaction.GroupInviteTransactionData; import org.qortal.data.transaction.TransactionData; @@ -11,6 +12,7 @@ import org.qortal.repository.Repository; import java.util.Collections; import java.util.List; +import java.util.Objects; public class GroupInviteTransaction extends Transaction { @@ -85,6 +87,16 @@ public class GroupInviteTransaction extends Transaction { if (admin.getConfirmedBalance(Asset.QORT) < this.groupInviteTransactionData.getFee()) return ValidationResult.NO_BALANCE; + // if null ownership group, then check for admin approval + if( this.repository.getBlockRepository().getBlockchainHeight() >= BlockChain.getInstance().getNullGroupMembershipHeight() ) { + String groupOwner = this.repository.getGroupRepository().getOwner(groupId); + boolean groupOwnedByNullAccount = Objects.equals(groupOwner, Group.NULL_OWNER_ADDRESS); + + // Require approval if transaction relates to a group owned by the null account + if (groupOwnedByNullAccount && !this.needsGroupApproval()) + return ValidationResult.GROUP_APPROVAL_REQUIRED; + } + return ValidationResult.OK; } diff --git a/src/main/java/org/qortal/transaction/GroupKickTransaction.java b/src/main/java/org/qortal/transaction/GroupKickTransaction.java index 3c426039..e13114fc 100644 --- a/src/main/java/org/qortal/transaction/GroupKickTransaction.java +++ b/src/main/java/org/qortal/transaction/GroupKickTransaction.java @@ -3,6 +3,7 @@ package org.qortal.transaction; import org.qortal.account.Account; import org.qortal.account.PublicKeyAccount; import org.qortal.asset.Asset; +import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; import org.qortal.data.group.GroupData; import org.qortal.data.transaction.GroupKickTransactionData; @@ -14,6 +15,7 @@ import org.qortal.repository.Repository; import java.util.Collections; import java.util.List; +import java.util.Objects; public class GroupKickTransaction extends Transaction { @@ -82,9 +84,26 @@ public class GroupKickTransaction extends Transaction { if (!admin.getAddress().equals(groupData.getOwner()) && groupRepository.adminExists(groupId, member.getAddress())) return ValidationResult.INVALID_GROUP_OWNER; - // Can't kick if not group's current owner - if (!admin.getAddress().equals(groupData.getOwner())) - return ValidationResult.INVALID_GROUP_OWNER; + if( this.repository.getBlockRepository().getBlockchainHeight() < BlockChain.getInstance().getNullGroupMembershipHeight() ) { + // Can't kick if not group's current owner + if (!admin.getAddress().equals(groupData.getOwner())) + return ValidationResult.INVALID_GROUP_OWNER; + } + // if( this.repository.getBlockRepository().getBlockchainHeight() >= BlockChain.getInstance().getNullGroupMembershipHeight() ) + else { + String groupOwner = this.repository.getGroupRepository().getOwner(groupId); + boolean groupOwnedByNullAccount = Objects.equals(groupOwner, Group.NULL_OWNER_ADDRESS); + + // if null ownership group, then check for admin approval + if(groupOwnedByNullAccount ) { + // Require approval if transaction relates to a group owned by the null account + if (!this.needsGroupApproval()) + return ValidationResult.GROUP_APPROVAL_REQUIRED; + } + // Can't kick if not group's current owner + else if (!admin.getAddress().equals(groupData.getOwner())) + return ValidationResult.INVALID_GROUP_OWNER; + } // Check creator has enough funds if (admin.getConfirmedBalance(Asset.QORT) < this.groupKickTransactionData.getFee()) diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index eb774252..f993194a 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -65,11 +65,11 @@ public abstract class Transaction { UPDATE_GROUP(23, true), ADD_GROUP_ADMIN(24, true), REMOVE_GROUP_ADMIN(25, true), - GROUP_BAN(26, false), - CANCEL_GROUP_BAN(27, false), - GROUP_KICK(28, false), - GROUP_INVITE(29, false), - CANCEL_GROUP_INVITE(30, false), + GROUP_BAN(26, true), + CANCEL_GROUP_BAN(27, true), + GROUP_KICK(28, true), + GROUP_INVITE(29, true), + CANCEL_GROUP_INVITE(30, true), JOIN_GROUP(31, false), LEAVE_GROUP(32, false), GROUP_APPROVAL(33, false), diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 70622061..ec27efd4 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -115,7 +115,8 @@ "groupMemberCheckHeight": 1902700, "fixBatchRewardHeight": 1945900, "adminsReplaceFoundersHeight": 9999999, - "onlineValidationFailSafeHeight": 9999999 + "onlineValidationFailSafeHeight": 9999999, + "nullGroupMembershipHeight": 9999999 }, "checkpoints": [ { "height": 1136300, "signature": "3BbwawEF2uN8Ni5ofpJXkukoU8ctAPxYoFB7whq9pKfBnjfZcpfEJT4R95NvBDoTP8WDyWvsUvbfHbcr9qSZuYpSKZjUQTvdFf6eqznHGEwhZApWfvXu6zjGCxYCp65F4jsVYYJjkzbjmkCg5WAwN5voudngA23kMK6PpTNygapCzXt" } diff --git a/src/test/java/org/qortal/test/group/DevGroupAdminTests.java b/src/test/java/org/qortal/test/group/DevGroupAdminTests.java index 41352323..14f2b87c 100644 --- a/src/test/java/org/qortal/test/group/DevGroupAdminTests.java +++ b/src/test/java/org/qortal/test/group/DevGroupAdminTests.java @@ -4,7 +4,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.block.Block; +import org.qortal.block.BlockChain; import org.qortal.data.transaction.*; +import org.qortal.group.Group; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -16,6 +19,8 @@ import org.qortal.test.common.transaction.TestTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.ValidationResult; +import java.util.List; + import static org.junit.Assert.*; /** @@ -40,8 +45,14 @@ import static org.junit.Assert.*; */ public class DevGroupAdminTests extends Common { + public static final int NULL_GROUP_MEMBERSHIP_HEIGHT = BlockChain.getInstance().getNullGroupMembershipHeight(); private static final int DEV_GROUP_ID = 1; + public static final String ALICE = "alice"; + public static final String BOB = "bob"; + public static final String CHLOE = "chloe"; + public static final String DILBERT = "dilbert"; + @Before public void beforeTest() throws DataException { Common.useDefaultSettings(); @@ -55,8 +66,8 @@ public class DevGroupAdminTests extends Common { @Test public void testGroupKickMember() throws DataException { try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); - PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); + PrivateKeyAccount alice = Common.getTestAccount(repository, ALICE); + PrivateKeyAccount bob = Common.getTestAccount(repository, BOB); // Dev group int groupId = DEV_GROUP_ID; @@ -80,16 +91,10 @@ public class DevGroupAdminTests extends Common { // Attempt to kick Bob result = groupKick(repository, alice, groupId, bob.getAddress()); - // Should be OK - assertEquals(ValidationResult.OK, result); + // Should not be OK, cannot kick member out of null owned group + assertNotSame(ValidationResult.OK, result); - // Confirm Bob no longer a member - assertFalse(isMember(repository, bob.getAddress(), groupId)); - - // Orphan last block - BlockUtils.orphanLastBlock(repository); - - // Confirm Bob now a member + // Confirm Bob remains a member assertTrue(isMember(repository, bob.getAddress(), groupId)); } } @@ -97,8 +102,8 @@ public class DevGroupAdminTests extends Common { @Test public void testGroupKickAdmin() throws DataException { try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); - PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); + PrivateKeyAccount alice = Common.getTestAccount(repository, ALICE); + PrivateKeyAccount bob = Common.getTestAccount(repository, BOB); // Dev group int groupId = DEV_GROUP_ID; @@ -123,7 +128,7 @@ public class DevGroupAdminTests extends Common { assertEquals("incorrect transaction approval status", Transaction.ApprovalStatus.PENDING, approvalStatus); // Have Alice approve Bob's approval-needed transaction - GroupUtils.approveTransaction(repository, "alice", addGroupAdminTransactionData.getSignature(), true); + GroupUtils.approveTransaction(repository, ALICE, addGroupAdminTransactionData.getSignature(), true); // Mint a block so that the transaction becomes approved BlockUtils.mintBlock(repository); @@ -167,8 +172,8 @@ public class DevGroupAdminTests extends Common { @Test public void testGroupBanMember() throws DataException { try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); - PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); + PrivateKeyAccount alice = Common.getTestAccount(repository, ALICE); + PrivateKeyAccount bob = Common.getTestAccount(repository, BOB); // Dev group int groupId = DEV_GROUP_ID; @@ -183,18 +188,13 @@ public class DevGroupAdminTests extends Common { // Attempt to ban Bob result = groupBan(repository, alice, groupId, bob.getAddress()); - // Should be OK - assertEquals(ValidationResult.OK, result); - - // Bob attempts to rejoin - result = joinGroup(repository, bob, groupId); - // Should NOT be OK + // Should not be OK, cannot ban someone from a null owned group assertNotSame(ValidationResult.OK, result); - // Orphan last block (Bob ban) - BlockUtils.orphanLastBlock(repository); - // Delete unconfirmed group-ban transaction - TransactionUtils.deleteUnconfirmedTransactions(repository); + // Bob attempts to join + result = joinGroup(repository, bob, groupId); + // Should be OK, but won't actually get him in the group + assertEquals(ValidationResult.OK, result); // Confirm Bob is not a member assertFalse(isMember(repository, bob.getAddress(), groupId)); @@ -204,65 +204,38 @@ public class DevGroupAdminTests extends Common { // Bob to join result = joinGroup(repository, bob, groupId); - // Should be OK - assertEquals(ValidationResult.OK, result); + // Should not be OK, bob should already be a member, he joined before the invite and + // the invite served as an approval + assertEquals(ValidationResult.ALREADY_GROUP_MEMBER, result); - // Confirm Bob now a member + // Confirm Bob now a member, now that he got an invite assertTrue(isMember(repository, bob.getAddress(), groupId)); // Attempt to ban Bob result = groupBan(repository, alice, groupId, bob.getAddress()); - // Should be OK - assertEquals(ValidationResult.OK, result); + // Should not be OK, because you can ban a member of a null owned group + assertNotSame(ValidationResult.OK, result); - // Confirm Bob no longer a member - assertFalse(isMember(repository, bob.getAddress(), groupId)); + // Confirm Bob is still a member + assertTrue(isMember(repository, bob.getAddress(), groupId)); // Bob attempts to rejoin result = joinGroup(repository, bob, groupId); - // Should NOT be OK + // Should NOT be OK, because he is already a member assertNotSame(ValidationResult.OK, result); // Cancel Bob's ban result = cancelGroupBan(repository, alice, groupId, bob.getAddress()); - // Should be OK - assertEquals(ValidationResult.OK, result); - - // Bob attempts to rejoin - result = joinGroup(repository, bob, groupId); - // Should be OK - assertEquals(ValidationResult.OK, result); - - // Orphan last block (Bob join) - BlockUtils.orphanLastBlock(repository); - // Delete unconfirmed join-group transaction - TransactionUtils.deleteUnconfirmedTransactions(repository); - - // Orphan last block (Cancel Bob ban) - BlockUtils.orphanLastBlock(repository); - // Delete unconfirmed cancel-ban transaction - TransactionUtils.deleteUnconfirmedTransactions(repository); - - // Bob attempts to rejoin - result = joinGroup(repository, bob, groupId); - // Should NOT be OK + // Should not be OK, because there was no ban to begin with assertNotSame(ValidationResult.OK, result); - - // Orphan last block (Bob ban) - BlockUtils.orphanLastBlock(repository); - // Delete unconfirmed group-ban transaction - TransactionUtils.deleteUnconfirmedTransactions(repository); - - // Confirm Bob now a member - assertTrue(isMember(repository, bob.getAddress(), groupId)); } } @Test public void testGroupBanAdmin() throws DataException { try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); - PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); + PrivateKeyAccount alice = Common.getTestAccount(repository, ALICE); + PrivateKeyAccount bob = Common.getTestAccount(repository, BOB); // Dev group int groupId = DEV_GROUP_ID; @@ -286,7 +259,7 @@ public class DevGroupAdminTests extends Common { assertEquals("incorrect transaction approval status", Transaction.ApprovalStatus.PENDING, approvalStatus); // Have Alice approve Bob's approval-needed transaction - GroupUtils.approveTransaction(repository, "alice", addGroupAdminTransactionData.getSignature(), true); + GroupUtils.approveTransaction(repository, ALICE, addGroupAdminTransactionData.getSignature(), true); // Mint a block so that the transaction becomes approved BlockUtils.mintBlock(repository); @@ -321,6 +294,302 @@ public class DevGroupAdminTests extends Common { } } + @Test + public void testAddAdmin2of3() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // establish accounts + PrivateKeyAccount alice = Common.getTestAccount(repository, ALICE); + PrivateKeyAccount bob = Common.getTestAccount(repository, BOB); + PrivateKeyAccount chloe = Common.getTestAccount(repository, CHLOE); + PrivateKeyAccount dilbert = Common.getTestAccount(repository, DILBERT); + + // assert admin statuses + assertEquals(2, repository.getGroupRepository().countGroupAdmins(DEV_GROUP_ID).intValue()); + assertTrue(isAdmin(repository, Group.NULL_OWNER_ADDRESS, DEV_GROUP_ID)); + assertTrue(isAdmin(repository, alice.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, bob.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, chloe.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, dilbert.getAddress(), DEV_GROUP_ID)); + + // confirm Bob is not a member + assertFalse(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + + // alice invites bob + ValidationResult result = groupInvite(repository, alice, DEV_GROUP_ID, bob.getAddress(), 3600); + assertSame(ValidationResult.OK, result); + + // bob joins + joinGroup(repository, bob, DEV_GROUP_ID); + + // confirm Bob is a member now, but still not an admin + assertTrue(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, bob.getAddress(), DEV_GROUP_ID)); + + // bob creates transaction to add himself as an admin + TransactionData addGroupAdminTransactionData1 = addGroupAdmin(repository, bob, DEV_GROUP_ID, bob.getAddress()); + + // bob creates add admin transaction for himself, alice signs which is 50% approval while 40% is needed + signForGroupApproval(repository, addGroupAdminTransactionData1, List.of(alice)); + + // assert 3 admins in group and bob is an admin now + assertEquals(3, repository.getGroupRepository().countGroupAdmins(DEV_GROUP_ID).intValue() ); + assertTrue(isAdmin(repository, bob.getAddress(), DEV_GROUP_ID)); + + // bob invites chloe + result = groupInvite(repository, bob, DEV_GROUP_ID, chloe.getAddress(), 3600); + assertSame(ValidationResult.OK, result); + + // chloe joins + joinGroup(repository, chloe, DEV_GROUP_ID); + + // confirm Chloe is a member now, but still not an admin + assertTrue(isMember(repository, chloe.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, chloe.getAddress(), DEV_GROUP_ID)); + + // chloe creates transaction to add herself as an admin + TransactionData addChloeAsGroupAdmin = addGroupAdmin(repository, chloe, DEV_GROUP_ID, chloe.getAddress()); + + // no one has signed, so it should be pending + Transaction.ApprovalStatus addChloeAsGroupAdminStatus1 = GroupUtils.getApprovalStatus(repository, addChloeAsGroupAdmin.getSignature()); + assertEquals( Transaction.ApprovalStatus.PENDING, addChloeAsGroupAdminStatus1); + + // signer 1 + Transaction.ApprovalStatus addChloeAsGroupAdminStatus2 = signForGroupApproval(repository, addChloeAsGroupAdmin, List.of(alice)); + + // 1 out of 3 has signed, so it should be pending, because it is less than 40% + assertEquals( Transaction.ApprovalStatus.PENDING, addChloeAsGroupAdminStatus2); + + // signer 2 + Transaction.ApprovalStatus addChloeAsGroupAdminStatus3 = signForGroupApproval(repository, addChloeAsGroupAdmin, List.of(bob)); + + // 2 out of 3 has signed, so it should be approved, because it is more than 40% + assertEquals( Transaction.ApprovalStatus.APPROVED, addChloeAsGroupAdminStatus3); + } + } + + @Test + public void testNullOwnershipMembership() throws DataException{ + try (final Repository repository = RepositoryManager.getRepository()) { + + Block block = BlockUtils.mintBlocks(repository, NULL_GROUP_MEMBERSHIP_HEIGHT); + assertEquals(NULL_GROUP_MEMBERSHIP_HEIGHT + 1, block.getBlockData().getHeight().intValue()); + + // establish accounts + PrivateKeyAccount alice = Common.getTestAccount(repository, ALICE); + PrivateKeyAccount bob = Common.getTestAccount(repository, BOB); + PrivateKeyAccount chloe = Common.getTestAccount(repository, CHLOE); + PrivateKeyAccount dilbert = Common.getTestAccount(repository, DILBERT); + + // assert admin statuses + assertEquals(2, repository.getGroupRepository().countGroupAdmins(DEV_GROUP_ID).intValue()); + assertTrue(isAdmin(repository, Group.NULL_OWNER_ADDRESS, DEV_GROUP_ID)); + assertTrue(isAdmin(repository, alice.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, bob.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, chloe.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, dilbert.getAddress(), DEV_GROUP_ID)); + + // confirm Bob is not a member + assertFalse(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + + // alice invites bob, alice signs which is 50% approval while 40% is needed + TransactionData createInviteTransactionData = createGroupInviteForGroupApproval(repository, alice, DEV_GROUP_ID, bob.getAddress(), 3600); + Transaction.ApprovalStatus bobsInviteStatus = signForGroupApproval(repository, createInviteTransactionData, List.of(alice)); + + // assert approval + assertEquals(Transaction.ApprovalStatus.APPROVED, bobsInviteStatus); + + // bob joins + joinGroup(repository, bob, DEV_GROUP_ID); + + // confirm Bob is a member now, but still not an admin + assertTrue(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, bob.getAddress(), DEV_GROUP_ID)); + + // bob creates transaction to add himself as an admin + TransactionData addGroupAdminTransactionData1 = addGroupAdmin(repository, bob, DEV_GROUP_ID, bob.getAddress()); + + // bob creates add admin transaction for himself, alice signs which is 50% approval while 40% is needed + signForGroupApproval(repository, addGroupAdminTransactionData1, List.of(alice)); + + // assert 3 admins in group and bob is an admin now + assertEquals(3, repository.getGroupRepository().countGroupAdmins(DEV_GROUP_ID).intValue()); + assertTrue(isAdmin(repository, bob.getAddress(), DEV_GROUP_ID)); + + // bob invites chloe, bob signs which is 33% approval while 40% is needed + TransactionData chloeInvite = createGroupInviteForGroupApproval(repository, bob, DEV_GROUP_ID, chloe.getAddress(), 3600); + Transaction.ApprovalStatus chloeInviteStatus = signForGroupApproval(repository, chloeInvite, List.of(bob)); + + // assert pending + assertEquals(Transaction.ApprovalStatus.PENDING, chloeInviteStatus); + + // alice signs which is 66% approval while 40% is needed + chloeInviteStatus = signForGroupApproval(repository, chloeInvite, List.of(alice)); + + // assert approval + assertEquals(Transaction.ApprovalStatus.APPROVED, chloeInviteStatus); + + // chloe joins + joinGroup(repository, chloe, DEV_GROUP_ID); + + // assert chloe is in the group + assertTrue(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + + // alice kicks chloe, alice signs which is 33% approval while 40% is needed + TransactionData chloeKick = createGroupKickForGroupApproval(repository, alice, DEV_GROUP_ID, chloe.getAddress(),"testing chloe kick"); + Transaction.ApprovalStatus chloeKickStatus = signForGroupApproval(repository, chloeKick, List.of(alice)); + + // assert pending + assertEquals(Transaction.ApprovalStatus.PENDING, chloeKickStatus); + + // assert chloe is still in the group + assertTrue(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + + // bob signs which is 66% approval while 40% is needed + chloeKickStatus = signForGroupApproval(repository, chloeKick, List.of(bob)); + + // assert approval + assertEquals(Transaction.ApprovalStatus.APPROVED, chloeKickStatus); + + // assert chloe is not in the group + assertFalse(isMember(repository, chloe.getAddress(), DEV_GROUP_ID)); + + // bob invites chloe, alice and bob signs which is 66% approval while 40% is needed + TransactionData chloeInviteAgain = createGroupInviteForGroupApproval(repository, bob, DEV_GROUP_ID, chloe.getAddress(), 3600); + Transaction.ApprovalStatus chloeInviteAgainStatus = signForGroupApproval(repository, chloeInviteAgain, List.of(alice, bob)); + + // assert approved + assertEquals(Transaction.ApprovalStatus.APPROVED, chloeInviteAgainStatus); + + // chloe joins again + joinGroup(repository, chloe, DEV_GROUP_ID); + + // assert chloe is in the group + assertTrue(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + + // alice bans chloe, alice signs which is 33% approval while 40% is needed + TransactionData chloeBan = createGroupBanForGroupApproval(repository, alice, DEV_GROUP_ID, chloe.getAddress(), "testing group ban", 3600); + Transaction.ApprovalStatus chloeBanStatus1 = signForGroupApproval(repository, chloeBan, List.of(alice)); + + // assert pending + assertEquals(Transaction.ApprovalStatus.PENDING, chloeBanStatus1); + + // bob signs which 66% approval while 40% is needed + Transaction.ApprovalStatus chloeBanStatus2 = signForGroupApproval(repository, chloeBan, List.of(bob)); + + // assert approved + assertEquals(Transaction.ApprovalStatus.APPROVED, chloeBanStatus2); + + // assert chloe is not in the group + assertFalse(isMember(repository, chloe.getAddress(), DEV_GROUP_ID)); + + // bob invites chloe, alice and bob signs which is 66% approval while 40% is needed + ValidationResult chloeInviteValidation = signAndImportGroupInvite(repository, bob, DEV_GROUP_ID, chloe.getAddress(), 3600); + + // assert banned status on invite attempt + assertEquals(ValidationResult.BANNED_FROM_GROUP, chloeInviteValidation); + + // bob cancel ban on chloe, bob signs which is 33% approval while 40% is needed + TransactionData chloeCancelBan = createCancelGroupBanForGroupApproval( repository, bob, DEV_GROUP_ID, chloe.getAddress()); + Transaction.ApprovalStatus chloeCancelBanStatus1 = signForGroupApproval(repository, chloeCancelBan, List.of(bob)); + + // assert pending + assertEquals(Transaction.ApprovalStatus.PENDING, chloeCancelBanStatus1); + + // alice signs which is 66% approval while 40% is needed + Transaction.ApprovalStatus chloeCancelBanStatus2 = signForGroupApproval(repository, chloeCancelBan, List.of(alice)); + + // assert approved + assertEquals(Transaction.ApprovalStatus.APPROVED, chloeCancelBanStatus2); + + // bob invites chloe, alice and bob signs which is 66% approval while 40% is needed + TransactionData chloeInvite4 = createGroupInviteForGroupApproval(repository, bob, DEV_GROUP_ID, chloe.getAddress(), 3600); + Transaction.ApprovalStatus chloeInvite4Status = signForGroupApproval(repository, chloeInvite4, List.of(alice, bob)); + + // assert approved + assertEquals(Transaction.ApprovalStatus.APPROVED, chloeInvite4Status); + + // chloe joins again + joinGroup(repository, chloe, DEV_GROUP_ID); + + // assert chloe is in the group + assertTrue(isMember(repository, chloe.getAddress(), DEV_GROUP_ID)); + + // bob invites dilbert, alice and bob signs which is 66% approval while 40% is needed + TransactionData dilbertInvite1 = createGroupInviteForGroupApproval(repository, bob, DEV_GROUP_ID, dilbert.getAddress(), 3600); + Transaction.ApprovalStatus dibertInviteStatus1 = signForGroupApproval(repository, dilbertInvite1, List.of(alice, bob)); + + // assert approved + assertEquals(Transaction.ApprovalStatus.APPROVED, dibertInviteStatus1); + + // alice cancels dilbert's invite, alice signs which is 33% approval while 40% is needed + TransactionData cancelDilbertInvite = createCancelInviteForGroupApproval(repository, alice, DEV_GROUP_ID, dilbert.getAddress()); + Transaction.ApprovalStatus cancelDilbertInviteStatus1 = signForGroupApproval(repository, cancelDilbertInvite, List.of(alice)); + + // assert pending + assertEquals(Transaction.ApprovalStatus.PENDING, cancelDilbertInviteStatus1); + + // dilbert joins before the group approves cancellation + joinGroup(repository, dilbert, DEV_GROUP_ID); + + // assert dilbert is in the group + assertTrue(isMember(repository, dilbert.getAddress(), DEV_GROUP_ID)); + + // alice kicks out dilbert, alice and bob sign which is 66% approval while 40% is needed + TransactionData kickDilbert = createGroupKickForGroupApproval(repository, alice, DEV_GROUP_ID, dilbert.getAddress(), "he is sneaky"); + Transaction.ApprovalStatus kickDilbertStatus = signForGroupApproval(repository, kickDilbert, List.of(alice, bob)); + + // assert approved + assertEquals(Transaction.ApprovalStatus.APPROVED, kickDilbertStatus); + + // assert dilbert is out of the group + assertFalse(isMember(repository, dilbert.getAddress(), DEV_GROUP_ID)); + + // bob invites dilbert again, alice and bob signs which is 66% approval while 40% is needed + TransactionData dilbertInvite2 = createGroupInviteForGroupApproval(repository, bob, DEV_GROUP_ID, dilbert.getAddress(), 3600); + Transaction.ApprovalStatus dibertInviteStatus2 = signForGroupApproval(repository, dilbertInvite2, List.of(alice, bob)); + + // assert approved + assertEquals(Transaction.ApprovalStatus.APPROVED, dibertInviteStatus2); + + // alice cancels dilbert's invite, alice and bob signs which is 66% approval while 40% is needed + TransactionData cancelDilbertInvite2 = createCancelInviteForGroupApproval(repository, alice, DEV_GROUP_ID, dilbert.getAddress()); + Transaction.ApprovalStatus cancelDilbertInviteStatus2 = signForGroupApproval(repository, cancelDilbertInvite2, List.of(alice, bob)); + + // assert approved + assertEquals(Transaction.ApprovalStatus.APPROVED, cancelDilbertInviteStatus2); + + // dilbert tries to join after the group approves cancellation + joinGroup(repository, dilbert, DEV_GROUP_ID); + + // assert dilbert is not in the group + assertFalse(isMember(repository, dilbert.getAddress(), DEV_GROUP_ID)); + } + } + + private Transaction.ApprovalStatus signForGroupApproval(Repository repository, TransactionData data, List signers) throws DataException { + + for (PrivateKeyAccount signer : signers) { + signTransactionDataForGroupApproval(repository, signer, data); + } + + BlockUtils.mintBlocks(repository, 2); + + // return approval status + return GroupUtils.getApprovalStatus(repository, data.getSignature()); + } + + private static void signTransactionDataForGroupApproval(Repository repository, PrivateKeyAccount signer, TransactionData transactionData) throws DataException { + byte[] reference = signer.getLastReference(); + long timestamp = repository.getTransactionRepository().fromSignature(reference).getTimestamp() + 1; + + BaseTransactionData baseTransactionData + = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, signer.getPublicKey(), GroupUtils.fee, null); + TransactionData groupApprovalTransactionData + = new GroupApprovalTransactionData(baseTransactionData, transactionData.getSignature(), true); + + TransactionUtils.signAndImportValid(repository, groupApprovalTransactionData, signer); + } private ValidationResult joinGroup(Repository repository, PrivateKeyAccount joiner, int groupId) throws DataException { JoinGroupTransactionData transactionData = new JoinGroupTransactionData(TestTransaction.generateBase(joiner), groupId); @@ -332,9 +601,31 @@ public class DevGroupAdminTests extends Common { return result; } - private void groupInvite(Repository repository, PrivateKeyAccount admin, int groupId, String invitee, int timeToLive) throws DataException { + private ValidationResult groupInvite(Repository repository, PrivateKeyAccount admin, int groupId, String invitee, int timeToLive) throws DataException { GroupInviteTransactionData transactionData = new GroupInviteTransactionData(TestTransaction.generateBase(admin), groupId, invitee, timeToLive); + ValidationResult result = TransactionUtils.signAndImport(repository, transactionData, admin); + + if (result == ValidationResult.OK) + BlockUtils.mintBlock(repository); + + return result; + } + + private TransactionData createGroupInviteForGroupApproval(Repository repository, PrivateKeyAccount admin, int groupId, String invitee, int timeToLive) throws DataException { + GroupInviteTransactionData transactionData = new GroupInviteTransactionData(TestTransaction.generateBase(admin, groupId), groupId, invitee, timeToLive); TransactionUtils.signAndMint(repository, transactionData, admin); + return transactionData; + } + + private TransactionData createCancelInviteForGroupApproval(Repository repository, PrivateKeyAccount admin, int groupId, String inviteeToCancel) throws DataException { + CancelGroupInviteTransactionData transactionData = new CancelGroupInviteTransactionData(TestTransaction.generateBase(admin, groupId), groupId, inviteeToCancel); + TransactionUtils.signAndMint(repository, transactionData, admin); + return transactionData; + } + + private ValidationResult signAndImportGroupInvite(Repository repository, PrivateKeyAccount admin, int groupId, String invitee, int timeToLive) throws DataException { + GroupInviteTransactionData transactionData = new GroupInviteTransactionData(TestTransaction.generateBase(admin, groupId), groupId, invitee, timeToLive); + return TransactionUtils.signAndImport(repository, transactionData, admin); } private ValidationResult groupKick(Repository repository, PrivateKeyAccount admin, int groupId, String member) throws DataException { @@ -347,6 +638,13 @@ public class DevGroupAdminTests extends Common { return result; } + private TransactionData createGroupKickForGroupApproval(Repository repository, PrivateKeyAccount admin, int groupId, String kicked, String reason) throws DataException { + GroupKickTransactionData transactionData = new GroupKickTransactionData(TestTransaction.generateBase(admin, groupId), groupId, kicked, reason); + TransactionUtils.signAndMint(repository, transactionData, admin); + + return transactionData; + } + private ValidationResult groupBan(Repository repository, PrivateKeyAccount admin, int groupId, String member) throws DataException { GroupBanTransactionData transactionData = new GroupBanTransactionData(TestTransaction.generateBase(admin), groupId, member, "testing", 0); ValidationResult result = TransactionUtils.signAndImport(repository, transactionData, admin); @@ -357,6 +655,13 @@ public class DevGroupAdminTests extends Common { return result; } + private TransactionData createGroupBanForGroupApproval(Repository repository, PrivateKeyAccount admin, int groupId, String banned, String reason, int timeToLive) throws DataException { + GroupBanTransactionData transactionData = new GroupBanTransactionData(TestTransaction.generateBase(admin, groupId), groupId, banned, reason, timeToLive); + TransactionUtils.signAndMint(repository, transactionData, admin); + + return transactionData; + } + private ValidationResult cancelGroupBan(Repository repository, PrivateKeyAccount admin, int groupId, String member) throws DataException { CancelGroupBanTransactionData transactionData = new CancelGroupBanTransactionData(TestTransaction.generateBase(admin), groupId, member); ValidationResult result = TransactionUtils.signAndImport(repository, transactionData, admin); @@ -367,6 +672,14 @@ public class DevGroupAdminTests extends Common { return result; } + private TransactionData createCancelGroupBanForGroupApproval(Repository repository, PrivateKeyAccount admin, int groupId, String unbanned ) throws DataException { + CancelGroupBanTransactionData transactionData = new CancelGroupBanTransactionData( TestTransaction.generateBase(admin, groupId), groupId, unbanned); + + TransactionUtils.signAndMint(repository, transactionData, admin); + + return transactionData; + } + private TransactionData addGroupAdmin(Repository repository, PrivateKeyAccount owner, int groupId, String member) throws DataException { AddGroupAdminTransactionData transactionData = new AddGroupAdminTransactionData(TestTransaction.generateBase(owner), groupId, member); transactionData.setTxGroupId(groupId); diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index 689b900b..b5666234 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -106,7 +106,8 @@ "removeOnlyMintWithNameHeight": 9999999999999, "fixBatchRewardHeight": 9999999999999, "adminsReplaceFoundersHeight": 9999999999999, - "onlineValidationFailSafeHeight": 9999999999999 + "onlineValidationFailSafeHeight": 9999999999999, + "nullGroupMembershipHeight": 20 }, "genesisInfo": { "version": 4, From c2ba9d142c5f973810814094b6361cbe146c6316 Mon Sep 17 00:00:00 2001 From: kennycud Date: Mon, 30 Dec 2024 12:15:27 -0800 Subject: [PATCH 11/24] crowetic's logging suggestions for the new reward distribution update --- src/main/java/org/qortal/block/Block.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 35e217c1..97b10ad1 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -2479,6 +2479,7 @@ public class Block { final long foundersShare = 1_00000000 - totalShares; BlockRewardCandidate rewardCandidate = new BlockRewardCandidate("Founders", foundersShare, founderDistributor); rewardCandidates.add(rewardCandidate); + LOGGER.info("logging foundersShare prior to reward modifications {}",foundersShare); } else if (this.blockData.getHeight() >= BlockChain.getInstance().getAdminsReplaceFoundersHeight()) { try (final Repository repository = RepositoryManager.getRepository()) { @@ -2502,6 +2503,8 @@ public class Block { distributeBlockRewardShare(distributionAmount, onlineMinterAdminAccounts, balanceChanges); long adminShare = 1_00000000 - totalShares; + LOGGER.info("initial total Shares: {}",totalShares); + LOGGER.info("logging adminShare after hardfork, this is the primary reward that will be split {}",adminShare); long minterAdminShare = adminShare / 2; BlockRewardCandidate minterAdminRewardCandidate @@ -2510,6 +2513,8 @@ public class Block { totalShares -= minterAdminShare; + LOGGER.info("MINTER ADMIN SHARE: {}",minterAdminShare); + // all dev admins List devAdminAddresses = groupRepository.getGroupAdmins(1).stream() @@ -2520,6 +2525,7 @@ public class Block { = (distributionAmount, balanceChanges) -> distributeToAccounts(distributionAmount, devAdminAddresses, balanceChanges); long devAdminShare = 1_00000000 - totalShares; + LOGGER.info("DEV ADMIN SHARE: {}",devAdminShare); BlockRewardCandidate devAdminRewardCandidate = new BlockRewardCandidate("Dev Admins", devAdminShare,devAdminDistributor); rewardCandidates.add(devAdminRewardCandidate); From ebc58c5c5cd2ac5a5d1ab46a9cba8a9c0d4ad9cd Mon Sep 17 00:00:00 2001 From: kennycud Date: Mon, 30 Dec 2024 16:01:53 -0800 Subject: [PATCH 12/24] qualified Groups table name, so it will be compatible with HSQLDB updated release which uses Groups for as a reserved word --- .../hsqldb/transaction/HSQLDBTransactionRepository.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index fe0b4d0b..d31011c9 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -1125,12 +1125,12 @@ public class HSQLDBTransactionRepository implements TransactionRepository { public List getApprovalPendingTransactions(int blockHeight) throws DataException { StringBuilder sql = new StringBuilder(512); sql.append("SELECT signature FROM Transactions " - + "JOIN Groups on Groups.group_id = Transactions.tx_group_id " + + "JOIN `Groups` g on g.group_id = Transactions.tx_group_id " + "WHERE Transactions.approval_status = "); // Enum int value safe to use literally sql.append(ApprovalStatus.PENDING.value); - sql.append(" AND Transactions.block_height < ? - Groups.min_block_delay"); + sql.append(" AND Transactions.block_height < ? - g.min_block_delay"); List transactions = new ArrayList<>(); @@ -1160,12 +1160,12 @@ public class HSQLDBTransactionRepository implements TransactionRepository { public List getApprovalExpiringTransactions(int blockHeight) throws DataException { StringBuilder sql = new StringBuilder(512); sql.append("SELECT signature FROM Transactions " - + "JOIN Groups on Groups.group_id = Transactions.tx_group_id " + + "JOIN `Groups` g on g.group_id = Transactions.tx_group_id " + "WHERE Transactions.approval_status = "); // Enum int value safe to use literally sql.append(ApprovalStatus.PENDING.value); - sql.append(" AND Transactions.block_height < ? - Groups.max_block_delay"); + sql.append(" AND Transactions.block_height < ? - g.max_block_delay"); List transactions = new ArrayList<>(); From 756f3a243d44b32a832d54b42c21f74e53c1b8ae Mon Sep 17 00:00:00 2001 From: kennycud Date: Mon, 30 Dec 2024 18:36:44 -0800 Subject: [PATCH 13/24] negate founder effective minting level for admins replace founders hardfork --- src/main/java/org/qortal/account/Account.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/account/Account.java b/src/main/java/org/qortal/account/Account.java index 99fa5217..daf8ad18 100644 --- a/src/main/java/org/qortal/account/Account.java +++ b/src/main/java/org/qortal/account/Account.java @@ -342,8 +342,13 @@ public class Account { return 0; // Founders are assigned a different effective minting level, as long as they have no penalty - if (Account.isFounder(accountData.getFlags()) && accountData.getBlocksMintedPenalty() == 0) - return BlockChain.getInstance().getFounderEffectiveMintingLevel(); + if (Account.isFounder(accountData.getFlags()) && accountData.getBlocksMintedPenalty() == 0) { + + // but only if it is before the admins replace founders hardfork + if( this.repository.getBlockRepository().getBlockchainHeight() < BlockChain.getInstance().getAdminsReplaceFoundersHeight() ) { + return BlockChain.getInstance().getFounderEffectiveMintingLevel(); + } + } return accountData.getLevel(); } From 278243f01cfe6d17f46ccf6d8c0ba20b84d1c556 Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 31 Dec 2024 13:54:20 -0800 Subject: [PATCH 14/24] rollback the negation of founder effective minting level, because I made it under the assumption that it was used for reward distributions when it is used for block signatures only --- src/main/java/org/qortal/account/Account.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/account/Account.java b/src/main/java/org/qortal/account/Account.java index daf8ad18..99fa5217 100644 --- a/src/main/java/org/qortal/account/Account.java +++ b/src/main/java/org/qortal/account/Account.java @@ -342,13 +342,8 @@ public class Account { return 0; // Founders are assigned a different effective minting level, as long as they have no penalty - if (Account.isFounder(accountData.getFlags()) && accountData.getBlocksMintedPenalty() == 0) { - - // but only if it is before the admins replace founders hardfork - if( this.repository.getBlockRepository().getBlockchainHeight() < BlockChain.getInstance().getAdminsReplaceFoundersHeight() ) { - return BlockChain.getInstance().getFounderEffectiveMintingLevel(); - } - } + if (Account.isFounder(accountData.getFlags()) && accountData.getBlocksMintedPenalty() == 0) + return BlockChain.getInstance().getFounderEffectiveMintingLevel(); return accountData.getLevel(); } From 8f6b55a98be71344c1084a2773c457016981f696 Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 31 Dec 2024 13:57:39 -0800 Subject: [PATCH 15/24] rollback the Groups table back quotes, because this only works with my testing environment and causes problems in production --- .../hsqldb/HSQLDBGroupRepository.java | 26 +++++++++---------- .../HSQLDBTransactionRepository.java | 8 +++--- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java index a364ed32..9c7521fc 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java @@ -24,7 +24,7 @@ public class HSQLDBGroupRepository implements GroupRepository { public GroupData fromGroupId(int groupId) throws DataException { String sql = "SELECT group_name, owner, description, created_when, updated_when, reference, is_open, " + "approval_threshold, min_block_delay, max_block_delay, creation_group_id, reduced_group_name " - + "FROM `Groups` WHERE group_id = ?"; + + "FROM Groups WHERE group_id = ?"; try (ResultSet resultSet = this.repository.checkedExecute(sql, groupId)) { if (resultSet == null) @@ -62,7 +62,7 @@ public class HSQLDBGroupRepository implements GroupRepository { public GroupData fromGroupName(String groupName) throws DataException { String sql = "SELECT group_id, owner, description, created_when, updated_when, reference, is_open, " + "approval_threshold, min_block_delay, max_block_delay, creation_group_id, reduced_group_name " - + "FROM `Groups` WHERE group_name = ?"; + + "FROM Groups WHERE group_name = ?"; try (ResultSet resultSet = this.repository.checkedExecute(sql, groupName)) { if (resultSet == null) @@ -99,7 +99,7 @@ public class HSQLDBGroupRepository implements GroupRepository { @Override public boolean groupExists(int groupId) throws DataException { try { - return this.repository.exists("`Groups`", "group_id = ?", groupId); + return this.repository.exists("Groups", "group_id = ?", groupId); } catch (SQLException e) { throw new DataException("Unable to check for group in repository", e); } @@ -108,7 +108,7 @@ public class HSQLDBGroupRepository implements GroupRepository { @Override public boolean groupExists(String groupName) throws DataException { try { - return this.repository.exists("`Groups`", "group_name = ?", groupName); + return this.repository.exists("Groups", "group_name = ?", groupName); } catch (SQLException e) { throw new DataException("Unable to check for group in repository", e); } @@ -117,7 +117,7 @@ public class HSQLDBGroupRepository implements GroupRepository { @Override public boolean reducedGroupNameExists(String reducedGroupName) throws DataException { try { - return this.repository.exists("`Groups`", "reduced_group_name = ?", reducedGroupName); + return this.repository.exists("Groups", "reduced_group_name = ?", reducedGroupName); } catch (SQLException e) { throw new DataException("Unable to check for reduced group name in repository", e); } @@ -129,7 +129,7 @@ public class HSQLDBGroupRepository implements GroupRepository { sql.append("SELECT group_id, owner, group_name, description, created_when, updated_when, reference, is_open, " + "approval_threshold, min_block_delay, max_block_delay, creation_group_id, reduced_group_name " - + "FROM `Groups` ORDER BY group_name"); + + "FROM Groups ORDER BY group_name"); if (reverse != null && reverse) sql.append(" DESC"); @@ -181,7 +181,7 @@ public class HSQLDBGroupRepository implements GroupRepository { sql.append("SELECT group_id, group_name, description, created_when, updated_when, reference, is_open, " + "approval_threshold, min_block_delay, max_block_delay, creation_group_id, reduced_group_name " - + "FROM `Groups` WHERE owner = ? ORDER BY group_name"); + + "FROM Groups WHERE owner = ? ORDER BY group_name"); if (reverse != null && reverse) sql.append(" DESC"); @@ -231,7 +231,7 @@ public class HSQLDBGroupRepository implements GroupRepository { StringBuilder sql = new StringBuilder(512); sql.append("SELECT group_id, owner, group_name, description, created_when, updated_when, reference, is_open, " - + "approval_threshold, min_block_delay, max_block_delay, creation_group_id, reduced_group_name, admin FROM `Groups` " + + "approval_threshold, min_block_delay, max_block_delay, creation_group_id, reduced_group_name, admin FROM Groups " + "JOIN GroupMembers USING (group_id) " + "LEFT OUTER JOIN GroupAdmins ON GroupAdmins.group_id = GroupMembers.group_id AND GroupAdmins.admin = GroupMembers.address " + "WHERE address = ? ORDER BY group_name"); @@ -289,7 +289,7 @@ public class HSQLDBGroupRepository implements GroupRepository { @Override public void save(GroupData groupData) throws DataException { - HSQLDBSaver saveHelper = new HSQLDBSaver("`Groups`"); + HSQLDBSaver saveHelper = new HSQLDBSaver("Groups"); saveHelper.bind("group_id", groupData.getGroupId()).bind("owner", groupData.getOwner()).bind("group_name", groupData.getGroupName()) .bind("description", groupData.getDescription()).bind("created_when", groupData.getCreated()).bind("updated_when", groupData.getUpdated()) @@ -302,7 +302,7 @@ public class HSQLDBGroupRepository implements GroupRepository { if (groupData.getGroupId() == null) { // Fetch new groupId - try (ResultSet resultSet = this.repository.checkedExecute("SELECT group_id FROM `Groups` WHERE reference = ?", groupData.getReference())) { + try (ResultSet resultSet = this.repository.checkedExecute("SELECT group_id FROM Groups WHERE reference = ?", groupData.getReference())) { if (resultSet == null) throw new DataException("Unable to fetch new group ID from repository"); @@ -318,7 +318,7 @@ public class HSQLDBGroupRepository implements GroupRepository { public void delete(int groupId) throws DataException { try { // Remove group - this.repository.delete("`Groups`", "group_id = ?", groupId); + this.repository.delete("Groups", "group_id = ?", groupId); } catch (SQLException e) { throw new DataException("Unable to delete group info from repository", e); } @@ -328,7 +328,7 @@ public class HSQLDBGroupRepository implements GroupRepository { public void delete(String groupName) throws DataException { try { // Remove group - this.repository.delete("`Groups`", "group_name = ?", groupName); + this.repository.delete("Groups", "group_name = ?", groupName); } catch (SQLException e) { throw new DataException("Unable to delete group info from repository", e); } @@ -338,7 +338,7 @@ public class HSQLDBGroupRepository implements GroupRepository { @Override public String getOwner(int groupId) throws DataException { - try (ResultSet resultSet = this.repository.checkedExecute("SELECT owner FROM `Groups` WHERE group_id = ?", groupId)) { + try (ResultSet resultSet = this.repository.checkedExecute("SELECT owner FROM Groups WHERE group_id = ?", groupId)) { if (resultSet == null) return null; diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index d31011c9..fe0b4d0b 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -1125,12 +1125,12 @@ public class HSQLDBTransactionRepository implements TransactionRepository { public List getApprovalPendingTransactions(int blockHeight) throws DataException { StringBuilder sql = new StringBuilder(512); sql.append("SELECT signature FROM Transactions " - + "JOIN `Groups` g on g.group_id = Transactions.tx_group_id " + + "JOIN Groups on Groups.group_id = Transactions.tx_group_id " + "WHERE Transactions.approval_status = "); // Enum int value safe to use literally sql.append(ApprovalStatus.PENDING.value); - sql.append(" AND Transactions.block_height < ? - g.min_block_delay"); + sql.append(" AND Transactions.block_height < ? - Groups.min_block_delay"); List transactions = new ArrayList<>(); @@ -1160,12 +1160,12 @@ public class HSQLDBTransactionRepository implements TransactionRepository { public List getApprovalExpiringTransactions(int blockHeight) throws DataException { StringBuilder sql = new StringBuilder(512); sql.append("SELECT signature FROM Transactions " - + "JOIN `Groups` g on g.group_id = Transactions.tx_group_id " + + "JOIN Groups on Groups.group_id = Transactions.tx_group_id " + "WHERE Transactions.approval_status = "); // Enum int value safe to use literally sql.append(ApprovalStatus.PENDING.value); - sql.append(" AND Transactions.block_height < ? - g.max_block_delay"); + sql.append(" AND Transactions.block_height < ? - Groups.max_block_delay"); List transactions = new ArrayList<>(); From f5d338435afeb869ef830c2c52029c6629c1293f Mon Sep 17 00:00:00 2001 From: kennycud Date: Thu, 2 Jan 2025 18:10:25 -0800 Subject: [PATCH 16/24] Since the Groups table is now named `Groups` with back ticks, it is now case-sensitive. Since it is now case-sensitive it needs to be in all caps, so when other SQL statements call on this table using the Groups without backticks it will be compatible. When Groups is used in a statement without back ticks or quotes it automatically gets converted into capital letters. --- .../hsqldb/HSQLDBDatabaseUpdates.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 0027e13a..ca55f3a8 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -454,41 +454,41 @@ public class HSQLDBDatabaseUpdates { case 12: // Groups - // NOTE: We need to set Groups to `Groups` here to avoid SQL Standard Keywords in HSQLDB v2.7.4 - stmt.execute("CREATE TABLE `Groups` (group_id GroupID, owner QortalAddress NOT NULL, group_name GroupName NOT NULL, " + // NOTE: We need to set Groups to `GROUPS` here to avoid SQL Standard Keywords in HSQLDB v2.7.4 + stmt.execute("CREATE TABLE `GROUPS` (group_id GroupID, owner QortalAddress NOT NULL, group_name GroupName NOT NULL, " + "created_when EpochMillis NOT NULL, updated_when EpochMillis, is_open BOOLEAN NOT NULL, " + "approval_threshold TINYINT NOT NULL, min_block_delay INTEGER NOT NULL, max_block_delay INTEGER NOT NULL, " + "reference Signature, creation_group_id GroupID, reduced_group_name GroupName NOT NULL, " + "description GenericDescription NOT NULL, PRIMARY KEY (group_id))"); // For finding groups by name - stmt.execute("CREATE INDEX GroupNameIndex on `Groups` (group_name)"); + stmt.execute("CREATE INDEX GroupNameIndex on `GROUPS` (group_name)"); // For finding groups by reduced name - stmt.execute("CREATE INDEX GroupReducedNameIndex on `Groups` (reduced_group_name)"); + stmt.execute("CREATE INDEX GroupReducedNameIndex on `GROUPS` (reduced_group_name)"); // For finding groups by owner - stmt.execute("CREATE INDEX GroupOwnerIndex ON `Groups` (owner)"); + stmt.execute("CREATE INDEX GroupOwnerIndex ON `GROUPS` (owner)"); // We need a corresponding trigger to make sure new group_id values are assigned sequentially starting from 1 - stmt.execute("CREATE TRIGGER Group_ID_Trigger BEFORE INSERT ON `Groups` " + stmt.execute("CREATE TRIGGER Group_ID_Trigger BEFORE INSERT ON `GROUPS` " + "REFERENCING NEW ROW AS new_row FOR EACH ROW WHEN (new_row.group_id IS NULL) " - + "SET new_row.group_id = (SELECT IFNULL(MAX(group_id) + 1, 1) FROM `Groups`)"); + + "SET new_row.group_id = (SELECT IFNULL(MAX(group_id) + 1, 1) FROM `GROUPS`)"); // Admins stmt.execute("CREATE TABLE GroupAdmins (group_id GroupID, admin QortalAddress, reference Signature NOT NULL, " - + "PRIMARY KEY (group_id, admin), FOREIGN KEY (group_id) REFERENCES `Groups` (group_id) ON DELETE CASCADE)"); + + "PRIMARY KEY (group_id, admin), FOREIGN KEY (group_id) REFERENCES `GROUPS` (group_id) ON DELETE CASCADE)"); // For finding groups by admin address stmt.execute("CREATE INDEX GroupAdminIndex ON GroupAdmins (admin)"); // Members stmt.execute("CREATE TABLE GroupMembers (group_id GroupID, address QortalAddress, " + "joined_when EpochMillis NOT NULL, reference Signature NOT NULL, " - + "PRIMARY KEY (group_id, address), FOREIGN KEY (group_id) REFERENCES `Groups` (group_id) ON DELETE CASCADE)"); + + "PRIMARY KEY (group_id, address), FOREIGN KEY (group_id) REFERENCES `GROUPS` (group_id) ON DELETE CASCADE)"); // For finding groups by member address stmt.execute("CREATE INDEX GroupMemberIndex ON GroupMembers (address)"); // Invites stmt.execute("CREATE TABLE GroupInvites (group_id GroupID, inviter QortalAddress, invitee QortalAddress, " + "expires_when EpochMillis, reference Signature, " - + "PRIMARY KEY (group_id, invitee), FOREIGN KEY (group_id) REFERENCES `Groups` (group_id) ON DELETE CASCADE)"); + + "PRIMARY KEY (group_id, invitee), FOREIGN KEY (group_id) REFERENCES `GROUPS` (group_id) ON DELETE CASCADE)"); // For finding invites sent by inviter stmt.execute("CREATE INDEX GroupInviteInviterIndex ON GroupInvites (inviter)"); // For finding invites by group @@ -504,7 +504,7 @@ public class HSQLDBDatabaseUpdates { // NULL expires_when means does not expire! stmt.execute("CREATE TABLE GroupBans (group_id GroupID, offender QortalAddress, admin QortalAddress NOT NULL, " + "banned_when EpochMillis NOT NULL, reason GenericDescription NOT NULL, expires_when EpochMillis, reference Signature NOT NULL, " - + "PRIMARY KEY (group_id, offender), FOREIGN KEY (group_id) REFERENCES `Groups` (group_id) ON DELETE CASCADE)"); + + "PRIMARY KEY (group_id, offender), FOREIGN KEY (group_id) REFERENCES `GROUPS` (group_id) ON DELETE CASCADE)"); // For expiry maintenance stmt.execute("CREATE INDEX GroupBanExpiryIndex ON GroupBans (expires_when)"); break; From 2392b7b15546ac07f99d9653a436f03f2550c78e Mon Sep 17 00:00:00 2001 From: kennycud Date: Sun, 5 Jan 2025 13:49:31 -0800 Subject: [PATCH 17/24] system info and database connection status access --- .../org/hsqldb/jdbc/HSQLDBPoolMonitored.java | 173 ++++++++++++++++++ .../org/qortal/api/model/DatasetStatus.java | 50 +++++ .../restricted/resource/AdminResource.java | 51 +++++- .../qortal/data/system/DbConnectionInfo.java | 35 ++++ .../org/qortal/data/system/SystemInfo.java | 49 +++++ .../hsqldb/HSQLDBRepositoryFactory.java | 27 ++- .../java/org/qortal/settings/Settings.java | 11 ++ 7 files changed, 394 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/hsqldb/jdbc/HSQLDBPoolMonitored.java create mode 100644 src/main/java/org/qortal/api/model/DatasetStatus.java create mode 100644 src/main/java/org/qortal/data/system/DbConnectionInfo.java create mode 100644 src/main/java/org/qortal/data/system/SystemInfo.java diff --git a/src/main/java/org/hsqldb/jdbc/HSQLDBPoolMonitored.java b/src/main/java/org/hsqldb/jdbc/HSQLDBPoolMonitored.java new file mode 100644 index 00000000..2037453c --- /dev/null +++ b/src/main/java/org/hsqldb/jdbc/HSQLDBPoolMonitored.java @@ -0,0 +1,173 @@ +package org.hsqldb.jdbc; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.hsqldb.jdbc.pool.JDBCPooledConnection; +import org.qortal.data.system.DbConnectionInfo; +import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; + +import javax.sql.ConnectionEvent; +import javax.sql.PooledConnection; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * Class HSQLDBPoolMonitored + * + * This class uses the same logic as HSQLDBPool. The only difference is it monitors the state of every connection + * to the database. This is used for debugging purposes only. + */ +public class HSQLDBPoolMonitored extends HSQLDBPool { + + private static final Logger LOGGER = LogManager.getLogger(HSQLDBRepositoryFactory.class); + + private static final String EMPTY = "Empty"; + private static final String AVAILABLE = "Available"; + private static final String ALLOCATED = "Allocated"; + + private ConcurrentHashMap infoByIndex; + + public HSQLDBPoolMonitored(int poolSize) { + super(poolSize); + + this.infoByIndex = new ConcurrentHashMap<>(poolSize); + } + + /** + * Tries to retrieve a new connection using the properties that have already been + * set. + * + * @return a connection to the data source, or null if no spare connections in pool + * @exception SQLException if a database access error occurs + */ + public Connection tryConnection() throws SQLException { + for (int i = 0; i < states.length(); i++) { + if (states.compareAndSet(i, RefState.available, RefState.allocated)) { + JDBCPooledConnection pooledConnection = connections[i]; + + if (pooledConnection == null) + // Probably shutdown situation + return null; + + infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), ALLOCATED)); + + return pooledConnection.getConnection(); + } + + if (states.compareAndSet(i, RefState.empty, RefState.allocated)) { + try { + JDBCPooledConnection pooledConnection = (JDBCPooledConnection) source.getPooledConnection(); + + if (pooledConnection == null) + // Probably shutdown situation + return null; + + pooledConnection.addConnectionEventListener(this); + pooledConnection.addStatementEventListener(this); + connections[i] = pooledConnection; + + infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), ALLOCATED)); + + return pooledConnection.getConnection(); + } catch (SQLException e) { + states.set(i, RefState.empty); + infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), EMPTY)); + } + } + } + + return null; + } + + public Connection getConnection() throws SQLException { + int var1 = 300; + if (this.source.loginTimeout != 0) { + var1 = this.source.loginTimeout * 10; + } + + if (this.closed) { + throw new SQLException("connection pool is closed"); + } else { + for(int var2 = 0; var2 < var1; ++var2) { + for(int var3 = 0; var3 < this.states.length(); ++var3) { + if (this.states.compareAndSet(var3, 1, 2)) { + infoByIndex.put(var3, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), ALLOCATED)); + return this.connections[var3].getConnection(); + } + + if (this.states.compareAndSet(var3, 0, 2)) { + try { + JDBCPooledConnection var4 = (JDBCPooledConnection)this.source.getPooledConnection(); + var4.addConnectionEventListener(this); + var4.addStatementEventListener(this); + this.connections[var3] = var4; + + infoByIndex.put(var3, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), ALLOCATED)); + + return this.connections[var3].getConnection(); + } catch (SQLException var6) { + this.states.set(var3, 0); + infoByIndex.put(var3, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), EMPTY)); + } + } + } + + try { + Thread.sleep(100L); + } catch (InterruptedException var5) { + } + } + + throw JDBCUtil.invalidArgument(); + } + } + + public void connectionClosed(ConnectionEvent event) { + PooledConnection connection = (PooledConnection) event.getSource(); + + for (int i = 0; i < connections.length; i++) { + if (connections[i] == connection) { + states.set(i, RefState.available); + infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), AVAILABLE)); + break; + } + } + } + + public void connectionErrorOccurred(ConnectionEvent event) { + PooledConnection connection = (PooledConnection) event.getSource(); + + for (int i = 0; i < connections.length; i++) { + if (connections[i] == connection) { + states.set(i, RefState.allocated); + connections[i] = null; + states.set(i, RefState.empty); + infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), EMPTY)); + break; + } + } + } + + public List getDbConnectionsStates() { + + return infoByIndex.values().stream() + .sorted(Comparator.comparingLong(DbConnectionInfo::getUpdated)) + .collect(Collectors.toList()); + } + + private int findConnectionIndex(ConnectionEvent connectionEvent) { + PooledConnection pooledConnection = (PooledConnection) connectionEvent.getSource(); + + for(int i = 0; i < this.connections.length; ++i) { + if (this.connections[i] == pooledConnection) { + return i; + } + } + + return -1; + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/model/DatasetStatus.java b/src/main/java/org/qortal/api/model/DatasetStatus.java new file mode 100644 index 00000000..b587be51 --- /dev/null +++ b/src/main/java/org/qortal/api/model/DatasetStatus.java @@ -0,0 +1,50 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class DatasetStatus { + + private String name; + + private long count; + + public DatasetStatus() {} + + public DatasetStatus(String name, long count) { + this.name = name; + this.count = count; + } + + public String getName() { + return name; + } + + public long getCount() { + return count; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DatasetStatus that = (DatasetStatus) o; + return count == that.count && Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name, count); + } + + @Override + public String toString() { + return "DatasetStatus{" + + "name='" + name + '\'' + + ", count=" + count + + '}'; + } +} diff --git a/src/main/java/org/qortal/api/restricted/resource/AdminResource.java b/src/main/java/org/qortal/api/restricted/resource/AdminResource.java index 279485bc..439904eb 100644 --- a/src/main/java/org/qortal/api/restricted/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/restricted/resource/AdminResource.java @@ -32,6 +32,7 @@ import org.qortal.controller.Synchronizer.SynchronizationResult; import org.qortal.controller.repository.BlockArchiveRebuilder; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; +import org.qortal.data.system.DbConnectionInfo; import org.qortal.network.Network; import org.qortal.network.Peer; import org.qortal.network.PeerAddress; @@ -40,6 +41,7 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; +import org.qortal.data.system.SystemInfo; import org.qortal.utils.Base58; import org.qortal.utils.NTP; @@ -52,6 +54,7 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -1064,4 +1067,50 @@ public class AdminResource { return "true"; } -} + @GET + @Path("/systeminfo") + @Operation( + summary = "System Information", + description = "System memory usage and available processors.", + responses = { + @ApiResponse( + description = "memory usage and available processors", + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = SystemInfo.class)) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + public SystemInfo getSystemInformation() { + + SystemInfo info + = new SystemInfo( + Runtime.getRuntime().freeMemory(), + Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(), + Runtime.getRuntime().totalMemory(), + Runtime.getRuntime().maxMemory(), + Runtime.getRuntime().availableProcessors()); + + return info; + } + + @GET + @Path("/dbstates") + @Operation( + summary = "Get DB States", + description = "Get DB States", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = DbConnectionInfo.class))) + ) + } + ) + public List getDbConnectionsStates() { + + try { + return Controller.REPOSITORY_FACTORY.getDbConnectionsStates(); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + return new ArrayList<>(0); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/data/system/DbConnectionInfo.java b/src/main/java/org/qortal/data/system/DbConnectionInfo.java new file mode 100644 index 00000000..0e42dc20 --- /dev/null +++ b/src/main/java/org/qortal/data/system/DbConnectionInfo.java @@ -0,0 +1,35 @@ +package org.qortal.data.system; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class DbConnectionInfo { + + private long updated; + + private String owner; + + private String state; + + public DbConnectionInfo() { + } + + public DbConnectionInfo(long timeOpened, String owner, String state) { + this.updated = timeOpened; + this.owner = owner; + this.state = state; + } + + public long getUpdated() { + return updated; + } + + public String getOwner() { + return owner; + } + + public String getState() { + return state; + } +} diff --git a/src/main/java/org/qortal/data/system/SystemInfo.java b/src/main/java/org/qortal/data/system/SystemInfo.java new file mode 100644 index 00000000..bf832194 --- /dev/null +++ b/src/main/java/org/qortal/data/system/SystemInfo.java @@ -0,0 +1,49 @@ +package org.qortal.data.system; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class SystemInfo { + + private long freeMemory; + + private long memoryInUse; + + private long totalMemory; + + private long maxMemory; + + private int availableProcessors; + + public SystemInfo() { + } + + public SystemInfo(long freeMemory, long memoryInUse, long totalMemory, long maxMemory, int availableProcessors) { + this.freeMemory = freeMemory; + this.memoryInUse = memoryInUse; + this.totalMemory = totalMemory; + this.maxMemory = maxMemory; + this.availableProcessors = availableProcessors; + } + + public long getFreeMemory() { + return freeMemory; + } + + public long getMemoryInUse() { + return memoryInUse; + } + + public long getTotalMemory() { + return totalMemory; + } + + public long getMaxMemory() { + return maxMemory; + } + + public int getAvailableProcessors() { + return availableProcessors; + } +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java index fdaf41a2..2ddabf8d 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java @@ -5,6 +5,8 @@ import org.apache.logging.log4j.Logger; import org.hsqldb.HsqlException; import org.hsqldb.error.ErrorCode; import org.hsqldb.jdbc.HSQLDBPool; +import org.hsqldb.jdbc.HSQLDBPoolMonitored; +import org.qortal.data.system.DbConnectionInfo; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryFactory; @@ -14,6 +16,8 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; import java.util.Properties; public class HSQLDBRepositoryFactory implements RepositoryFactory { @@ -57,7 +61,13 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory { HSQLDBRepository.attemptRecovery(connectionUrl, "backup"); } - this.connectionPool = new HSQLDBPool(Settings.getInstance().getRepositoryConnectionPoolSize()); + if(Settings.getInstance().isConnectionPoolMonitorEnabled()) { + this.connectionPool = new HSQLDBPoolMonitored(Settings.getInstance().getRepositoryConnectionPoolSize()); + } + else { + this.connectionPool = new HSQLDBPool(Settings.getInstance().getRepositoryConnectionPoolSize()); + } + this.connectionPool.setUrl(this.connectionUrl); Properties properties = new Properties(); @@ -153,4 +163,19 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory { return HSQLDBRepository.isDeadlockException(e); } + /** + * Get Connection States + * + * Get the database connection states, if database connection pool monitoring is enabled. + * + * @return the connection states if enabled, otherwise an empty list + */ + public List getDbConnectionsStates() { + if( Settings.getInstance().isConnectionPoolMonitorEnabled() ) { + return ((HSQLDBPoolMonitored) this.connectionPool).getDbConnectionsStates(); + } + else { + return new ArrayList<>(0); + } + } } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index deee0075..3a0d17bb 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -501,6 +501,13 @@ public class Settings { */ private boolean rewardRecordingOnly = true; + /** + * Is The Connection Monitored? + * + * Is the database connection pooled monitored? + */ + private boolean connectionPoolMonitorEnabled = false; + // Domain mapping public static class ThreadLimit { private String messageType; @@ -1322,4 +1329,8 @@ public class Settings { public boolean isRewardRecordingOnly() { return rewardRecordingOnly; } + + public boolean isConnectionPoolMonitorEnabled() { + return connectionPoolMonitorEnabled; + } } From d9a7648d36760da820d04994336d51cc4e9b318f Mon Sep 17 00:00:00 2001 From: kennycud Date: Sun, 5 Jan 2025 15:59:09 -0800 Subject: [PATCH 18/24] access to decoded online accounts by block --- .../qortal/api/resource/BlocksResource.java | 52 +++++++++- .../data/block/DecodedOnlineAccountData.java | 85 ++++++++++++++++ src/main/java/org/qortal/utils/Blocks.java | 99 +++++++++++++++++++ 3 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/qortal/data/block/DecodedOnlineAccountData.java create mode 100644 src/main/java/org/qortal/utils/Blocks.java diff --git a/src/main/java/org/qortal/api/resource/BlocksResource.java b/src/main/java/org/qortal/api/resource/BlocksResource.java index ff0bb979..0203bafc 100644 --- a/src/main/java/org/qortal/api/resource/BlocksResource.java +++ b/src/main/java/org/qortal/api/resource/BlocksResource.java @@ -19,6 +19,8 @@ import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountData; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; +import org.qortal.data.block.DecodedOnlineAccountData; +import org.qortal.data.network.OnlineAccountData; import org.qortal.data.transaction.TransactionData; import org.qortal.repository.BlockArchiveReader; import org.qortal.repository.DataException; @@ -27,6 +29,7 @@ import org.qortal.repository.RepositoryManager; import org.qortal.transform.TransformationException; import org.qortal.transform.block.BlockTransformer; import org.qortal.utils.Base58; +import org.qortal.utils.Blocks; import org.qortal.utils.Triple; import javax.servlet.http.HttpServletRequest; @@ -45,6 +48,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Set; @Path("/blocks") @Tag(name = "Blocks") @@ -889,4 +893,50 @@ public class BlocksResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } } -} + + @GET + @Path("/onlineaccounts/{height}") + @Operation( + summary = "Get online accounts for block", + description = "Returns the online accounts who submitted signatures for this block", + responses = { + @ApiResponse( + description = "online accounts", + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = DecodedOnlineAccountData.class + ) + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE + }) + public Set getOnlineAccounts(@PathParam("height") int height) { + + try (final Repository repository = RepositoryManager.getRepository()) { + + // get block from database + BlockData blockData = repository.getBlockRepository().fromHeight(height); + + // if block data is not in the database, then try the archive + if (blockData == null) { + blockData = repository.getBlockArchiveRepository().fromHeight(height); + + // if the block is not in the database or the archive, then the block is unknown + if( blockData == null ) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + } + + Set onlineAccounts = Blocks.getDecodedOnlineAccountsForBlock(repository, blockData); + + return onlineAccounts; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/data/block/DecodedOnlineAccountData.java b/src/main/java/org/qortal/data/block/DecodedOnlineAccountData.java new file mode 100644 index 00000000..a2ecc1ca --- /dev/null +++ b/src/main/java/org/qortal/data/block/DecodedOnlineAccountData.java @@ -0,0 +1,85 @@ +package org.qortal.data.block; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; + +// All properties to be converted to JSON via JAX-RS +@XmlAccessorType(XmlAccessType.FIELD) +public class DecodedOnlineAccountData { + + private long onlineTimestamp; + private String minter; + private String recipient; + private int sharePercent; + private boolean minterGroupMember; + private String name; + private int level; + + public DecodedOnlineAccountData() { + } + + public DecodedOnlineAccountData(long onlineTimestamp, String minter, String recipient, int sharePercent, boolean minterGroupMember, String name, int level) { + this.onlineTimestamp = onlineTimestamp; + this.minter = minter; + this.recipient = recipient; + this.sharePercent = sharePercent; + this.minterGroupMember = minterGroupMember; + this.name = name; + this.level = level; + } + + public long getOnlineTimestamp() { + return onlineTimestamp; + } + + public String getMinter() { + return minter; + } + + public String getRecipient() { + return recipient; + } + + public int getSharePercent() { + return sharePercent; + } + + public boolean isMinterGroupMember() { + return minterGroupMember; + } + + public String getName() { + return name; + } + + public int getLevel() { + return level; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DecodedOnlineAccountData that = (DecodedOnlineAccountData) o; + return onlineTimestamp == that.onlineTimestamp && sharePercent == that.sharePercent && minterGroupMember == that.minterGroupMember && level == that.level && Objects.equals(minter, that.minter) && Objects.equals(recipient, that.recipient) && Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(onlineTimestamp, minter, recipient, sharePercent, minterGroupMember, name, level); + } + + @Override + public String toString() { + return "DecodedOnlineAccountData{" + + "onlineTimestamp=" + onlineTimestamp + + ", minter='" + minter + '\'' + + ", recipient='" + recipient + '\'' + + ", sharePercent=" + sharePercent + + ", minterGroupMember=" + minterGroupMember + + ", name='" + name + '\'' + + ", level=" + level + + '}'; + } +} diff --git a/src/main/java/org/qortal/utils/Blocks.java b/src/main/java/org/qortal/utils/Blocks.java new file mode 100644 index 00000000..54ad86da --- /dev/null +++ b/src/main/java/org/qortal/utils/Blocks.java @@ -0,0 +1,99 @@ +package org.qortal.utils; + +import io.druid.extendedset.intset.ConciseSet; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.block.BlockChain; +import org.qortal.data.account.AddressLevelPairing; +import org.qortal.data.account.RewardShareData; +import org.qortal.data.block.BlockData; +import org.qortal.data.block.DecodedOnlineAccountData; +import org.qortal.data.group.GroupMemberData; +import org.qortal.data.naming.NameData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transform.block.BlockTransformer; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Class Blocks + * + * Methods for block related logic. + */ +public class Blocks { + + private static final Logger LOGGER = LogManager.getLogger(Blocks.class); + + /** + * Get Decode Online Accounts For Block + * + * @param repository the data repository + * @param blockData the block data + * + * @return the online accounts set to the block + * + * @throws DataException + */ + public static Set getDecodedOnlineAccountsForBlock(Repository repository, BlockData blockData) throws DataException { + try { + // get all online account indices from block + ConciseSet onlineAccountIndices = BlockTransformer.decodeOnlineAccounts(blockData.getEncodedOnlineAccounts()); + + // get online reward shares from the online accounts on the block + List onlineRewardShares = repository.getAccountRepository().getRewardSharesByIndexes(onlineAccountIndices.toArray()); + + // online timestamp for block + long onlineTimestamp = blockData.getOnlineAccountsTimestamp(); + Set onlineAccounts = new HashSet<>(); + + // all minting group member addresses + List mintingGroupAddresses + = repository.getGroupRepository() + .getGroupMembers(BlockChain.getInstance().getMintingGroupId()).stream() + .map(GroupMemberData::getMember) + .collect(Collectors.toList()); + + // all names, indexed by address + Map nameByAddress + = repository.getNameRepository() + .getAllNames().stream() + .collect(Collectors.toMap(NameData::getOwner, NameData::getName)); + + // all accounts at level 1 or higher, indexed by address + Map levelByAddress + = repository.getAccountRepository().getAddressLevelPairings(1).stream() + .collect(Collectors.toMap(AddressLevelPairing::getAddress, AddressLevelPairing::getLevel)); + + // for each reward share where the minter is online, + // construct the data object and add it to the return list + for (RewardShareData onlineRewardShare : onlineRewardShares) { + String minter = onlineRewardShare.getMinter(); + DecodedOnlineAccountData onlineAccountData + = new DecodedOnlineAccountData( + onlineTimestamp, + minter, + onlineRewardShare.getRecipient(), + onlineRewardShare.getSharePercent(), + mintingGroupAddresses.contains(minter), + nameByAddress.get(minter), + levelByAddress.get(minter) + ); + + onlineAccounts.add(onlineAccountData); + } + + return onlineAccounts; + } catch (DataException e) { + throw e; + } catch (Exception e ) { + LOGGER.error(e.getMessage(), e); + + return new HashSet<>(0); + } + } +} \ No newline at end of file From 2805bb8364d138d84e5dee2e1f66f6856b283aeb Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 7 Jan 2025 13:20:39 -0800 Subject: [PATCH 19/24] corrected an arithmetic error --- src/main/java/org/qortal/block/Block.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 97b10ad1..c254fa7c 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -2511,7 +2511,7 @@ public class Block { = new BlockRewardCandidate("Minter Admins", minterAdminShare, minterAdminDistributor); rewardCandidates.add(minterAdminRewardCandidate); - totalShares -= minterAdminShare; + totalShares += minterAdminShare; LOGGER.info("MINTER ADMIN SHARE: {}",minterAdminShare); From a8a8904ebff595b3974521278882ee30d802ce24 Mon Sep 17 00:00:00 2001 From: kennycud Date: Wed, 8 Jan 2025 16:19:38 -0800 Subject: [PATCH 20/24] removed the NULL account from the dev admin reward distribution and added some fail safes in case the admin groups are empty --- src/main/java/org/qortal/block/Block.java | 38 ++++++++++++++++------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index c254fa7c..e22d7993 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -26,6 +26,7 @@ import org.qortal.data.block.BlockTransactionData; import org.qortal.data.group.GroupAdminData; import org.qortal.data.network.OnlineAccountData; import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; import org.qortal.repository.*; import org.qortal.settings.Settings; import org.qortal.transaction.AtTransaction; @@ -2497,21 +2498,28 @@ public class Block { .filter(expandedAccount -> minterAdmins.contains(expandedAccount.getMintingAccount().getAddress())) .collect(Collectors.toList()); - BlockRewardDistributor minterAdminDistributor - = (distributionAmount, balanceChanges) - -> - distributeBlockRewardShare(distributionAmount, onlineMinterAdminAccounts, balanceChanges); + long minterAdminShare; - long adminShare = 1_00000000 - totalShares; - LOGGER.info("initial total Shares: {}",totalShares); - LOGGER.info("logging adminShare after hardfork, this is the primary reward that will be split {}",adminShare); + if( onlineMinterAdminAccounts.isEmpty() ) { + minterAdminShare = 0; + } + else { + BlockRewardDistributor minterAdminDistributor + = (distributionAmount, balanceChanges) + -> + distributeBlockRewardShare(distributionAmount, onlineMinterAdminAccounts, balanceChanges); - long minterAdminShare = adminShare / 2; - BlockRewardCandidate minterAdminRewardCandidate - = new BlockRewardCandidate("Minter Admins", minterAdminShare, minterAdminDistributor); - rewardCandidates.add(minterAdminRewardCandidate); + long adminShare = 1_00000000 - totalShares; + LOGGER.info("initial total Shares: {}", totalShares); + LOGGER.info("logging adminShare after hardfork, this is the primary reward that will be split {}", adminShare); - totalShares += minterAdminShare; + minterAdminShare = adminShare / 2; + BlockRewardCandidate minterAdminRewardCandidate + = new BlockRewardCandidate("Minter Admins", minterAdminShare, minterAdminDistributor); + rewardCandidates.add(minterAdminRewardCandidate); + + totalShares += minterAdminShare; + } LOGGER.info("MINTER ADMIN SHARE: {}",minterAdminShare); @@ -2521,6 +2529,10 @@ public class Block { .map(GroupAdminData::getAdmin) .collect(Collectors.toList()); + LOGGER.info("Removing NULL Account Address, Dev Admin Count = {}", devAdminAddresses.size()); + devAdminAddresses.removeIf( address -> Group.NULL_OWNER_ADDRESS.equals(address) ); + LOGGER.info("Removed NULL Account Address, Dev Admin Count = {}", devAdminAddresses.size()); + BlockRewardDistributor devAdminDistributor = (distributionAmount, balanceChanges) -> distributeToAccounts(distributionAmount, devAdminAddresses, balanceChanges); @@ -2548,6 +2560,8 @@ public class Block { */ public static long distributeToAccounts(long distributionAmount, List accountAddressess, Map balanceChanges) { + if( accountAddressess.isEmpty() ) return 0; + long distibutionShare = distributionAmount / accountAddressess.size(); for(String accountAddress : accountAddressess ) { From 70f4ff4fb36c5136061b7ce5f731ba37463f7309 Mon Sep 17 00:00:00 2001 From: kennycud Date: Sat, 11 Jan 2025 18:20:28 -0800 Subject: [PATCH 21/24] ignore level for reward share feature hard fork --- src/main/java/org/qortal/account/Account.java | 14 +++++++++++++- src/main/java/org/qortal/block/BlockChain.java | 7 ++++++- src/main/resources/blockchain.json | 3 ++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/account/Account.java b/src/main/java/org/qortal/account/Account.java index 99fa5217..856b79ef 100644 --- a/src/main/java/org/qortal/account/Account.java +++ b/src/main/java/org/qortal/account/Account.java @@ -216,7 +216,16 @@ public class Account { String myAddress = accountData.getAddress(); int blockchainHeight = this.repository.getBlockRepository().getBlockchainHeight(); - int levelToMint = BlockChain.getInstance().getMinAccountLevelToMint(); + + int levelToMint; + + if( blockchainHeight >= BlockChain.getInstance().getIgnoreLevelForRewardShareHeight() ) { + levelToMint = 0; + } + else { + levelToMint = BlockChain.getInstance().getMinAccountLevelToMint(); + } + int level = accountData.getLevel(); int groupIdToMint = BlockChain.getInstance().getMintingGroupId(); int nameCheckHeight = BlockChain.getInstance().getOnlyMintWithNameHeight(); @@ -306,6 +315,9 @@ public class Account { if (Account.isFounder(accountData.getFlags()) && accountData.getBlocksMintedPenalty() == 0) return true; + if( this.repository.getBlockRepository().getBlockchainHeight() >= BlockChain.getInstance().getIgnoreLevelForRewardShareHeight() ) + return true; + return false; } diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 42da0b3a..358874c5 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -91,7 +91,8 @@ public class BlockChain { fixBatchRewardHeight, adminsReplaceFoundersHeight, onlineValidationFailSafeHeight, - nullGroupMembershipHeight + nullGroupMembershipHeight, + ignoreLevelForRewardShareHeight } // Custom transaction fees @@ -677,6 +678,10 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.nullGroupMembershipHeight.name()).intValue(); } + public int getIgnoreLevelForRewardShareHeight() { + return this.featureTriggers.get(FeatureTrigger.ignoreLevelForRewardShareHeight.name()).intValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index ec27efd4..2cbacd97 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -116,7 +116,8 @@ "fixBatchRewardHeight": 1945900, "adminsReplaceFoundersHeight": 9999999, "onlineValidationFailSafeHeight": 9999999, - "nullGroupMembershipHeight": 9999999 + "nullGroupMembershipHeight": 9999999, + "ignoreLevelForRewardShareHeight": 9999999 }, "checkpoints": [ { "height": 1136300, "signature": "3BbwawEF2uN8Ni5ofpJXkukoU8ctAPxYoFB7whq9pKfBnjfZcpfEJT4R95NvBDoTP8WDyWvsUvbfHbcr9qSZuYpSKZjUQTvdFf6eqznHGEwhZApWfvXu6zjGCxYCp65F4jsVYYJjkzbjmkCg5WAwN5voudngA23kMK6PpTNygapCzXt" } From 69cba78d94a5ef9696930804978a5336dc137103 Mon Sep 17 00:00:00 2001 From: kennycud Date: Sat, 11 Jan 2025 19:01:13 -0800 Subject: [PATCH 22/24] exclude blocked implementation completion --- .../repository/hsqldb/HSQLDBCacheUtils.java | 5 ++++- .../test/repository/HSQLDBCacheUtilsTests.java | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java index 6e8dc8a8..24e2da56 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java @@ -180,7 +180,10 @@ public class HSQLDBCacheUtils { Optional reverse) { // retain only candidates with names - Stream stream = candidates.stream().filter(candidate -> candidate.name != null); + Stream stream = candidates.stream().filter(candidate -> candidate.name != null ); + + if(exclude.isPresent()) + stream = stream.filter( candidate -> !exclude.get().get().contains( candidate.name )); // filter by service if( service.isPresent() ) diff --git a/src/test/java/org/qortal/test/repository/HSQLDBCacheUtilsTests.java b/src/test/java/org/qortal/test/repository/HSQLDBCacheUtilsTests.java index 7812dbd7..7a75dea7 100644 --- a/src/test/java/org/qortal/test/repository/HSQLDBCacheUtilsTests.java +++ b/src/test/java/org/qortal/test/repository/HSQLDBCacheUtilsTests.java @@ -399,7 +399,7 @@ public class HSQLDBCacheUtilsTests { } @Test - public void testExcludeBlockedPositive() { + public void testExcludeBlockedNegative() { ArbitraryResourceData data = new ArbitraryResourceData(); data.name = "Joe"; @@ -413,6 +413,21 @@ public class HSQLDBCacheUtilsTests { ); } + @Test + public void testExcludeBlockedPositive() { + + ArbitraryResourceData data = new ArbitraryResourceData(); + data.name = "Joe"; + + Supplier> supplier = () -> List.of("Joe"); + + filterListByMap( + List.of(data), + NAME_LEVEL, new HashMap<>(Map.of(EXCLUDE_BLOCKED, supplier)), + 0 + ); + } + @Test public void testIncludeMetadataPositive() { From b2dbcbb6033787de57a1c9f8223c30d44797ec9f Mon Sep 17 00:00:00 2001 From: kennycud Date: Mon, 13 Jan 2025 13:52:17 -0800 Subject: [PATCH 23/24] made adjustments to support the ignore level feature trigger and removed the fail-safe feature trigger since the ignore level feature trigger now satisfies it implicitly --- src/main/java/org/qortal/block/Block.java | 45 +++++++++---------- .../java/org/qortal/block/BlockChain.java | 5 --- src/main/resources/blockchain.json | 1 - 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index e22d7993..21cbddc4 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -418,7 +418,9 @@ public class Block { onlineAccounts.removeIf(a -> a.getNonce() == null || a.getNonce() < 0); // After feature trigger, remove any online accounts that are level 0 - if (height >= BlockChain.getInstance().getOnlineAccountMinterLevelValidationHeight()) { + // but only if they are before the ignore level feature trigger + if (height < BlockChain.getInstance().getIgnoreLevelForRewardShareHeight() && + height >= BlockChain.getInstance().getOnlineAccountMinterLevelValidationHeight()) { onlineAccounts.removeIf(a -> { try { return Account.getRewardShareEffectiveMintingLevel(repository, a.getPublicKey()) == 0; @@ -1161,34 +1163,31 @@ public class Block { if (onlineRewardShares == null) return ValidationResult.ONLINE_ACCOUNT_UNKNOWN; - // After feature trigger, require all online account minters to be greater than level 0 - if (this.getBlockData().getHeight() >= BlockChain.getInstance().getOnlineAccountMinterLevelValidationHeight()) { - if( this.blockData.getHeight() < BlockChain.getInstance().getOnlineValidationFailSafeHeight()) { - List expandedAccounts + // After feature trigger, require all online account minters to be greater than level 0, + // but only if it is before the feature trigger where we ignore level again + if (this.blockData.getHeight() < BlockChain.getInstance().getIgnoreLevelForRewardShareHeight() && + this.getBlockData().getHeight() >= BlockChain.getInstance().getOnlineAccountMinterLevelValidationHeight()) { + List expandedAccounts = this.getExpandedAccounts().stream() - .filter(expandedAccount -> expandedAccount.isMinterMember) - .collect(Collectors.toList()); + .filter(expandedAccount -> expandedAccount.isMinterMember) + .collect(Collectors.toList()); - for (ExpandedAccount account : expandedAccounts) { - if (account.getMintingAccount().getEffectiveMintingLevel() == 0) + for (ExpandedAccount account : expandedAccounts) { + if (account.getMintingAccount().getEffectiveMintingLevel() == 0) + return ValidationResult.ONLINE_ACCOUNTS_INVALID; + + if (this.getBlockData().getHeight() >= BlockChain.getInstance().getFixBatchRewardHeight()) { + if (!account.isMinterMember) return ValidationResult.ONLINE_ACCOUNTS_INVALID; - - if (this.getBlockData().getHeight() >= BlockChain.getInstance().getFixBatchRewardHeight()) { - if (!account.isMinterMember) - return ValidationResult.ONLINE_ACCOUNTS_INVALID; - } } } - // this.blockData.getHeight() >= BlockChain.getInstance().getOnlineValidationFailSafeHeight() - else { - Optional anyInvalidAccount + } + else if (this.blockData.getHeight() >= BlockChain.getInstance().getIgnoreLevelForRewardShareHeight()){ + Optional anyInvalidAccount = this.getExpandedAccounts().stream() - .filter( - account -> account.getEffectiveMintingLevel() == 0 || - !account.isMinterMember) - .findAny(); - if( anyInvalidAccount.isPresent() ) return ValidationResult.ONLINE_ACCOUNTS_INVALID; - } + .filter(account -> !account.isMinterMember) + .findAny(); + if( anyInvalidAccount.isPresent() ) return ValidationResult.ONLINE_ACCOUNTS_INVALID; } // If block is past a certain age then we simply assume the signatures were correct diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 358874c5..ef20abe8 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -90,7 +90,6 @@ public class BlockChain { groupMemberCheckHeight, fixBatchRewardHeight, adminsReplaceFoundersHeight, - onlineValidationFailSafeHeight, nullGroupMembershipHeight, ignoreLevelForRewardShareHeight } @@ -670,10 +669,6 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.adminsReplaceFoundersHeight.name()).intValue(); } - public int getOnlineValidationFailSafeHeight() { - return this.featureTriggers.get(FeatureTrigger.onlineValidationFailSafeHeight.name()).intValue(); - } - public int getNullGroupMembershipHeight() { return this.featureTriggers.get(FeatureTrigger.nullGroupMembershipHeight.name()).intValue(); } diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 2cbacd97..f1a7e76a 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -115,7 +115,6 @@ "groupMemberCheckHeight": 1902700, "fixBatchRewardHeight": 1945900, "adminsReplaceFoundersHeight": 9999999, - "onlineValidationFailSafeHeight": 9999999, "nullGroupMembershipHeight": 9999999, "ignoreLevelForRewardShareHeight": 9999999 }, From 72f0194487072e0a07d9fa3102489e27e495ce87 Mon Sep 17 00:00:00 2001 From: kennycud Date: Fri, 17 Jan 2025 19:31:13 -0800 Subject: [PATCH 24/24] get admin query fix and hardfork --- .../java/org/qortal/block/BlockChain.java | 7 ++++++- src/main/java/org/qortal/group/Group.java | 8 ++++++- .../qortal/repository/GroupRepository.java | 2 ++ .../hsqldb/HSQLDBGroupRepository.java | 17 ++++++++++++++- src/main/resources/blockchain.json | 3 ++- .../qortal/test/group/DevGroupAdminTests.java | 21 +++++++++++++++++++ src/test/resources/test-chain-v2.json | 5 +++-- 7 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index ef20abe8..1468fbc3 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -91,7 +91,8 @@ public class BlockChain { fixBatchRewardHeight, adminsReplaceFoundersHeight, nullGroupMembershipHeight, - ignoreLevelForRewardShareHeight + ignoreLevelForRewardShareHeight, + adminQueryFixHeight } // Custom transaction fees @@ -677,6 +678,10 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.ignoreLevelForRewardShareHeight.name()).intValue(); } + public int getAdminQueryFixHeight() { + return this.featureTriggers.get(FeatureTrigger.adminQueryFixHeight.name()).intValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { diff --git a/src/main/java/org/qortal/group/Group.java b/src/main/java/org/qortal/group/Group.java index 59e32545..765b86de 100644 --- a/src/main/java/org/qortal/group/Group.java +++ b/src/main/java/org/qortal/group/Group.java @@ -2,6 +2,7 @@ package org.qortal.group; import org.qortal.account.Account; import org.qortal.account.PublicKeyAccount; +import org.qortal.block.BlockChain; import org.qortal.controller.Controller; import org.qortal.crypto.Crypto; import org.qortal.data.group.*; @@ -150,7 +151,12 @@ public class Group { // Adminship private GroupAdminData getAdmin(String admin) throws DataException { - return groupRepository.getAdmin(this.groupData.getGroupId(), admin); + if( repository.getBlockRepository().getBlockchainHeight() < BlockChain.getInstance().getAdminQueryFixHeight()) { + return groupRepository.getAdminFaulty(this.groupData.getGroupId(), admin); + } + else { + return groupRepository.getAdmin(this.groupData.getGroupId(), admin); + } } private boolean adminExists(String admin) throws DataException { diff --git a/src/main/java/org/qortal/repository/GroupRepository.java b/src/main/java/org/qortal/repository/GroupRepository.java index 49427b02..4e39333c 100644 --- a/src/main/java/org/qortal/repository/GroupRepository.java +++ b/src/main/java/org/qortal/repository/GroupRepository.java @@ -48,6 +48,8 @@ public interface GroupRepository { // Group Admins + public GroupAdminData getAdminFaulty(int groupId, String address) throws DataException; + public GroupAdminData getAdmin(int groupId, String address) throws DataException; public boolean adminExists(int groupId, String address) throws DataException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java index 9c7521fc..a15582e2 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java @@ -351,7 +351,7 @@ public class HSQLDBGroupRepository implements GroupRepository { // Group Admins @Override - public GroupAdminData getAdmin(int groupId, String address) throws DataException { + public GroupAdminData getAdminFaulty(int groupId, String address) throws DataException { try (ResultSet resultSet = this.repository.checkedExecute("SELECT admin, reference FROM GroupAdmins WHERE group_id = ?", groupId)) { if (resultSet == null) return null; @@ -365,6 +365,21 @@ public class HSQLDBGroupRepository implements GroupRepository { } } + @Override + public GroupAdminData getAdmin(int groupId, String address) throws DataException { + try (ResultSet resultSet = this.repository.checkedExecute("SELECT admin, reference FROM GroupAdmins WHERE group_id = ? AND admin = ?", groupId, address)) { + if (resultSet == null) + return null; + + String admin = resultSet.getString(1); + byte[] reference = resultSet.getBytes(2); + + return new GroupAdminData(groupId, admin, reference); + } catch (SQLException e) { + throw new DataException("Unable to fetch group admin from repository", e); + } + } + @Override public boolean adminExists(int groupId, String address) throws DataException { try { diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index f1a7e76a..5e7383b8 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -116,7 +116,8 @@ "fixBatchRewardHeight": 1945900, "adminsReplaceFoundersHeight": 9999999, "nullGroupMembershipHeight": 9999999, - "ignoreLevelForRewardShareHeight": 9999999 + "ignoreLevelForRewardShareHeight": 9999999, + "adminQueryFixHeight": 9999999 }, "checkpoints": [ { "height": 1136300, "signature": "3BbwawEF2uN8Ni5ofpJXkukoU8ctAPxYoFB7whq9pKfBnjfZcpfEJT4R95NvBDoTP8WDyWvsUvbfHbcr9qSZuYpSKZjUQTvdFf6eqznHGEwhZApWfvXu6zjGCxYCp65F4jsVYYJjkzbjmkCg5WAwN5voudngA23kMK6PpTNygapCzXt" } diff --git a/src/test/java/org/qortal/test/group/DevGroupAdminTests.java b/src/test/java/org/qortal/test/group/DevGroupAdminTests.java index 14f2b87c..38ad0c53 100644 --- a/src/test/java/org/qortal/test/group/DevGroupAdminTests.java +++ b/src/test/java/org/qortal/test/group/DevGroupAdminTests.java @@ -6,6 +6,7 @@ import org.junit.Test; import org.qortal.account.PrivateKeyAccount; import org.qortal.block.Block; import org.qortal.block.BlockChain; +import org.qortal.data.group.GroupAdminData; import org.qortal.data.transaction.*; import org.qortal.group.Group; import org.qortal.repository.DataException; @@ -567,6 +568,26 @@ public class DevGroupAdminTests extends Common { } } + @Test + public void testGetAdmin() throws DataException{ + try (final Repository repository = RepositoryManager.getRepository()) { + + // establish accounts + PrivateKeyAccount alice = Common.getTestAccount(repository, ALICE); + PrivateKeyAccount bob = Common.getTestAccount(repository, BOB); + + GroupAdminData aliceAdminData = repository.getGroupRepository().getAdmin(DEV_GROUP_ID, alice.getAddress()); + + assertNotNull(aliceAdminData); + assertEquals( alice.getAddress(), aliceAdminData.getAdmin() ); + assertEquals( DEV_GROUP_ID, aliceAdminData.getGroupId()); + + GroupAdminData bobAdminData = repository.getGroupRepository().getAdmin(DEV_GROUP_ID, bob.getAddress()); + + assertNull(bobAdminData); + } + } + private Transaction.ApprovalStatus signForGroupApproval(Repository repository, TransactionData data, List signers) throws DataException { for (PrivateKeyAccount signer : signers) { diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index b5666234..4d1d6240 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -106,8 +106,9 @@ "removeOnlyMintWithNameHeight": 9999999999999, "fixBatchRewardHeight": 9999999999999, "adminsReplaceFoundersHeight": 9999999999999, - "onlineValidationFailSafeHeight": 9999999999999, - "nullGroupMembershipHeight": 20 + "ignoreLevelForRewardShareHeight": 9999999999999, + "nullGroupMembershipHeight": 20, + "adminQueryFixHeight": 9999999999999 }, "genesisInfo": { "version": 4,