diff --git a/pom.xml b/pom.xml index 71424e58..82e0f42f 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 4.7.0 + 4.7.1 jar UTF-8 diff --git a/src/main/java/org/qortal/account/Account.java b/src/main/java/org/qortal/account/Account.java index 856b79ef..722e70da 100644 --- a/src/main/java/org/qortal/account/Account.java +++ b/src/main/java/org/qortal/account/Account.java @@ -14,6 +14,7 @@ import org.qortal.repository.NameRepository; import org.qortal.repository.Repository; import org.qortal.settings.Settings; import org.qortal.utils.Base58; +import org.qortal.utils.Groups; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -227,7 +228,7 @@ public class Account { } int level = accountData.getLevel(); - int groupIdToMint = BlockChain.getInstance().getMintingGroupId(); + List groupIdsToMint = Groups.getGroupIdsToMint( BlockChain.getInstance(), blockchainHeight ); int nameCheckHeight = BlockChain.getInstance().getOnlyMintWithNameHeight(); int groupCheckHeight = BlockChain.getInstance().getGroupMemberCheckHeight(); int removeNameCheckHeight = BlockChain.getInstance().getRemoveOnlyMintWithNameHeight(); @@ -261,9 +262,9 @@ public class Account { if (blockchainHeight >= groupCheckHeight && blockchainHeight < removeNameCheckHeight) { List myName = nameRepository.getNamesByOwner(myAddress); if (Account.isFounder(accountData.getFlags())) { - return accountData.getBlocksMintedPenalty() == 0 && !myName.isEmpty() && (isGroupValidated || groupRepository.memberExists(groupIdToMint, myAddress)); + return accountData.getBlocksMintedPenalty() == 0 && !myName.isEmpty() && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress)); } else { - return level >= levelToMint && !myName.isEmpty() && (isGroupValidated || groupRepository.memberExists(groupIdToMint, myAddress)); + return level >= levelToMint && !myName.isEmpty() && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress)); } } @@ -272,9 +273,9 @@ public class Account { // Account's address is a member of the minter group if (blockchainHeight >= removeNameCheckHeight) { if (Account.isFounder(accountData.getFlags())) { - return accountData.getBlocksMintedPenalty() == 0 && (isGroupValidated || groupRepository.memberExists(groupIdToMint, myAddress)); + return accountData.getBlocksMintedPenalty() == 0 && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress)); } else { - return level >= levelToMint && (isGroupValidated || groupRepository.memberExists(groupIdToMint, myAddress)); + return level >= levelToMint && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress)); } } diff --git a/src/main/java/org/qortal/api/ApiService.java b/src/main/java/org/qortal/api/ApiService.java index fbef50d3..2cebe8e5 100644 --- a/src/main/java/org/qortal/api/ApiService.java +++ b/src/main/java/org/qortal/api/ApiService.java @@ -194,6 +194,7 @@ public class ApiService { context.addServlet(AdminStatusWebSocket.class, "/websockets/admin/status"); context.addServlet(BlocksWebSocket.class, "/websockets/blocks"); + context.addServlet(DataMonitorSocket.class, "/websockets/datamonitor"); context.addServlet(ActiveChatsWebSocket.class, "/websockets/chat/active/*"); context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages"); context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers"); diff --git a/src/main/java/org/qortal/api/model/CrossChainTradeLedgerEntry.java b/src/main/java/org/qortal/api/model/CrossChainTradeLedgerEntry.java new file mode 100644 index 00000000..34f8fc57 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainTradeLedgerEntry.java @@ -0,0 +1,72 @@ +package org.qortal.api.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import org.qortal.data.crosschain.CrossChainTradeData; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainTradeLedgerEntry { + + private String market; + + private String currency; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long quantity; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long feeAmount; + + private String feeCurrency; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long totalPrice; + + private long tradeTimestamp; + + protected CrossChainTradeLedgerEntry() { + /* For JAXB */ + } + + public CrossChainTradeLedgerEntry(String market, String currency, long quantity, long feeAmount, String feeCurrency, long totalPrice, long tradeTimestamp) { + this.market = market; + this.currency = currency; + this.quantity = quantity; + this.feeAmount = feeAmount; + this.feeCurrency = feeCurrency; + this.totalPrice = totalPrice; + this.tradeTimestamp = tradeTimestamp; + } + + public String getMarket() { + return market; + } + + public String getCurrency() { + return currency; + } + + public long getQuantity() { + return quantity; + } + + public long getFeeAmount() { + return feeAmount; + } + + public String getFeeCurrency() { + return feeCurrency; + } + + public long getTotalPrice() { + return totalPrice; + } + + public long getTradeTimestamp() { + return tradeTimestamp; + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 754c3467..a6f44373 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -33,9 +33,13 @@ import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; import org.qortal.controller.arbitrary.ArbitraryMetadataManager; import org.qortal.data.account.AccountData; import org.qortal.data.arbitrary.ArbitraryCategoryInfo; +import org.qortal.data.arbitrary.ArbitraryDataIndexDetail; +import org.qortal.data.arbitrary.ArbitraryDataIndexScoreKey; +import org.qortal.data.arbitrary.ArbitraryDataIndexScorecard; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.arbitrary.ArbitraryResourceStatus; +import org.qortal.data.arbitrary.IndexCache; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; @@ -69,8 +73,11 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; @Path("/arbitrary") @Tag(name = "Arbitrary") @@ -172,6 +179,7 @@ public class ArbitraryResource { @Parameter(description = "Name (searches name field only)") @QueryParam("name") List names, @Parameter(description = "Title (searches title metadata field only)") @QueryParam("title") String title, @Parameter(description = "Description (searches description metadata field only)") @QueryParam("description") String description, + @Parameter(description = "Keyword (searches description metadata field by keywords)") @QueryParam("keywords") List keywords, @Parameter(description = "Prefix only (if true, only the beginning of fields are matched)") @QueryParam("prefix") Boolean prefixOnly, @Parameter(description = "Exact match names only (if true, partial name matches are excluded)") @QueryParam("exactmatchnames") Boolean exactMatchNamesOnly, @Parameter(description = "Default resources (without identifiers) only") @QueryParam("default") Boolean defaultResource, @@ -212,7 +220,7 @@ public class ArbitraryResource { } List resources = repository.getArbitraryRepository() - .searchArbitraryResources(service, query, identifier, names, title, description, usePrefixOnly, + .searchArbitraryResources(service, query, identifier, names, title, description, keywords, usePrefixOnly, exactMatchNames, defaultRes, mode, minLevel, followedOnly, excludeBlocked, includeMetadata, includeStatus, before, after, limit, offset, reverse); @@ -1185,6 +1193,90 @@ public class ArbitraryResource { } } + @GET + @Path("/indices") + @Operation( + summary = "Find matching arbitrary resource indices", + description = "", + responses = { + @ApiResponse( + description = "indices", + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = ArbitraryDataIndexScorecard.class + ) + ) + ) + ) + } + ) + public List searchIndices(@QueryParam("terms") String[] terms) { + + List indices = new ArrayList<>(); + + // get index details for each term + for( String term : terms ) { + List details = IndexCache.getInstance().getIndicesByTerm().get(term); + + if( details != null ) { + indices.addAll(details); + } + } + + // sum up the scores for each index with identical attributes + Map scoreForKey + = indices.stream() + .collect( + Collectors.groupingBy( + index -> new ArbitraryDataIndexScoreKey(index.name, index.category, index.link), + Collectors.summingDouble(detail -> 1.0 / detail.rank) + ) + ); + + // create scorecards for each index group and put them in descending order by score + List scorecards + = scoreForKey.entrySet().stream().map( + entry + -> + new ArbitraryDataIndexScorecard( + entry.getValue(), + entry.getKey().name, + entry.getKey().category, + entry.getKey().link) + ) + .sorted(Comparator.comparingDouble(ArbitraryDataIndexScorecard::getScore).reversed()) + .collect(Collectors.toList()); + + return scorecards; + } + + @GET + @Path("/indices/{name}/{idPrefix}") + @Operation( + summary = "Find matching arbitrary resource indices for a registered name and identifier prefix", + description = "", + responses = { + @ApiResponse( + description = "indices", + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = ArbitraryDataIndexDetail.class + ) + ) + ) + ) + } + ) + public List searchIndicesByName(@PathParam("name") String name, @PathParam("idPrefix") String idPrefix) { + + return + IndexCache.getInstance().getIndicesByIssuer() + .getOrDefault(name, new ArrayList<>(0)).stream() + .filter( indexDetail -> indexDetail.indexIdentifer.startsWith(idPrefix)) + .collect(Collectors.toList()); + } // Shared methods diff --git a/src/main/java/org/qortal/api/resource/CrossChainResource.java b/src/main/java/org/qortal/api/resource/CrossChainResource.java index 748dcbe4..3f7acf68 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainResource.java @@ -10,11 +10,13 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; +import org.glassfish.jersey.media.multipart.ContentDisposition; import org.qortal.api.ApiError; import org.qortal.api.ApiErrors; import org.qortal.api.ApiExceptionFactory; import org.qortal.api.Security; import org.qortal.api.model.CrossChainCancelRequest; +import org.qortal.api.model.CrossChainTradeLedgerEntry; import org.qortal.api.model.CrossChainTradeSummary; import org.qortal.controller.tradebot.TradeBot; import org.qortal.crosschain.ACCT; @@ -44,10 +46,14 @@ import org.qortal.utils.Base58; import org.qortal.utils.ByteArray; import org.qortal.utils.NTP; +import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import javax.ws.rs.*; import javax.ws.rs.core.Context; +import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; +import java.io.IOException; import java.util.*; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -61,6 +67,13 @@ public class CrossChainResource { @Context HttpServletRequest request; + @Context + HttpServletResponse response; + + @Context + ServletContext context; + + @GET @Path("/tradeoffers") @Operation( @@ -258,11 +271,11 @@ public class CrossChainResource { example = "1597310000000" ) @QueryParam("minimumTimestamp") Long minimumTimestamp, @Parameter( - description = "Optionally filter by buyer Qortal address" - ) @QueryParam("buyerAddress") String buyerAddress, + description = "Optionally filter by buyer Qortal public key" + ) @QueryParam("buyerPublicKey") String buyerPublicKey58, @Parameter( - description = "Optionally filter by seller Qortal address" - ) @QueryParam("sellerAddress") String sellerAddress, + description = "Optionally filter by seller Qortal public key" + ) @QueryParam("sellerPublicKey") String sellerPublicKey58, @Parameter( ref = "limit") @QueryParam("limit") Integer limit, @Parameter( ref = "offset" ) @QueryParam("offset") Integer offset, @Parameter( ref = "reverse" ) @QueryParam("reverse") Boolean reverse) { @@ -274,6 +287,10 @@ public class CrossChainResource { if (minimumTimestamp != null && minimumTimestamp <= 0) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + // Decode public keys + byte[] buyerPublicKey = decodePublicKey(buyerPublicKey58); + byte[] sellerPublicKey = decodePublicKey(sellerPublicKey58); + final Boolean isFinished = Boolean.TRUE; try (final Repository repository = RepositoryManager.getRepository()) { @@ -304,7 +321,7 @@ public class CrossChainResource { byte[] codeHash = acctInfo.getKey().value; ACCT acct = acctInfo.getValue().get(); - List atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, buyerAddress, sellerAddress, + List atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, buyerPublicKey, sellerPublicKey, isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumFinalHeight, limit, offset, reverse); @@ -343,6 +360,120 @@ public class CrossChainResource { } } + /** + * Decode Public Key + * + * @param publicKey58 the public key in a string + * + * @return the public key in bytes + */ + private byte[] decodePublicKey(String publicKey58) { + + if( publicKey58 == null ) return null; + if( publicKey58.isEmpty() ) return new byte[0]; + + byte[] publicKey; + try { + publicKey = Base58.decode(publicKey58); + } catch (NumberFormatException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY, e); + } + + // Correct size for public key? + if (publicKey.length != Transformer.PUBLIC_KEY_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + + return publicKey; + } + + @GET + @Path("/ledger/{publicKey}") + @Operation( + summary = "Accounting entries for all trades.", + description = "Returns accounting entries for all completed cross-chain trades", + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + type = "string", + format = "byte" + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public HttpServletResponse getLedgerEntries( + @PathParam("publicKey") String publicKey58, + @Parameter( + description = "Only return trades that completed on/after this timestamp (milliseconds since epoch)", + example = "1597310000000" + ) @QueryParam("minimumTimestamp") Long minimumTimestamp) { + + byte[] publicKey = decodePublicKey(publicKey58); + + // minimumTimestamp (if given) needs to be positive + if (minimumTimestamp != null && minimumTimestamp <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + try (final Repository repository = RepositoryManager.getRepository()) { + Integer minimumFinalHeight = null; + + if (minimumTimestamp != null) { + minimumFinalHeight = repository.getBlockRepository().getHeightFromTimestamp(minimumTimestamp); + // If not found in the block repository it will return either 0 or 1 + if (minimumFinalHeight == 0 || minimumFinalHeight == 1) { + // Try the archive + minimumFinalHeight = repository.getBlockArchiveRepository().getHeightFromTimestamp(minimumTimestamp); + } + + if (minimumFinalHeight == 0) + // We don't have any blocks since minimumTimestamp, let alone trades, so nothing to return + return response; + + // height returned from repository is for block BEFORE timestamp + // but we want trades AFTER timestamp so bump height accordingly + minimumFinalHeight++; + } + + List crossChainTradeLedgerEntries = new ArrayList<>(); + + Map> acctsByCodeHash = SupportedBlockchain.getAcctMap(); + + // collect ledger entries for each ACCT + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); + + // collect buys and sells + CrossChainUtils.collectLedgerEntries(publicKey, repository, minimumFinalHeight, crossChainTradeLedgerEntries, codeHash, acct, true); + CrossChainUtils.collectLedgerEntries(publicKey, repository, minimumFinalHeight, crossChainTradeLedgerEntries, codeHash, acct, false); + } + + crossChainTradeLedgerEntries.sort((a, b) -> Longs.compare(a.getTradeTimestamp(), b.getTradeTimestamp())); + + response.setStatus(HttpServletResponse.SC_OK); + response.setContentType("text/csv"); + response.setHeader( + HttpHeaders.CONTENT_DISPOSITION, + ContentDisposition + .type("attachment") + .fileName(CrossChainUtils.createLedgerFileName(Crypto.toAddress(publicKey))) + .build() + .toString() + ); + + CrossChainUtils.writeToLedger( response.getWriter(), crossChainTradeLedgerEntries); + + return response; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (IOException e) { + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return response; + } + } + @GET @Path("/price/{blockchain}") @Operation( diff --git a/src/main/java/org/qortal/api/resource/CrossChainUtils.java b/src/main/java/org/qortal/api/resource/CrossChainUtils.java index 802faca1..ddd1d2d6 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainUtils.java +++ b/src/main/java/org/qortal/api/resource/CrossChainUtils.java @@ -10,21 +10,36 @@ import org.bitcoinj.script.ScriptBuilder; import org.bouncycastle.util.Strings; import org.json.simple.JSONObject; +import org.qortal.api.model.CrossChainTradeLedgerEntry; import org.qortal.api.model.crosschain.BitcoinyTBDRequest; import org.qortal.crosschain.*; import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; import org.qortal.data.crosschain.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; +import org.qortal.utils.Amounts; import org.qortal.utils.BitTwiddling; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.Writer; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.*; import java.util.stream.Collectors; public class CrossChainUtils { + public static final String QORT_CURRENCY_CODE = "QORT"; private static final Logger LOGGER = LogManager.getLogger(CrossChainUtils.class); public static final String CORE_API_CALL = "Core API Call"; + public static final String QORTAL_EXCHANGE_LABEL = "Qortal"; public static ServerConfigurationInfo buildServerConfigurationInfo(Bitcoiny blockchain) { @@ -632,4 +647,128 @@ public class CrossChainUtils { byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); } + + /** + * Write To Ledger + * + * @param writer the writer to the ledger + * @param entries the entries to write to the ledger + * + * @throws IOException + */ + public static void writeToLedger(Writer writer, List entries) throws IOException { + + BufferedWriter bufferedWriter = new BufferedWriter(writer); + + StringJoiner header = new StringJoiner(","); + header.add("Market"); + header.add("Currency"); + header.add("Quantity"); + header.add("Commission Paid"); + header.add("Commission Currency"); + header.add("Total Price"); + header.add("Date Time"); + header.add("Exchange"); + + bufferedWriter.append(header.toString()); + + DateFormat dateFormatter = new SimpleDateFormat("yyyyMMdd HH:mm"); + dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); + + for( CrossChainTradeLedgerEntry entry : entries ) { + StringJoiner joiner = new StringJoiner(","); + + joiner.add(entry.getMarket()); + joiner.add(entry.getCurrency()); + joiner.add(String.valueOf(Amounts.prettyAmount(entry.getQuantity()))); + joiner.add(String.valueOf(Amounts.prettyAmount(entry.getFeeAmount()))); + joiner.add(entry.getFeeCurrency()); + joiner.add(String.valueOf(Amounts.prettyAmount(entry.getTotalPrice()))); + joiner.add(dateFormatter.format(new Date(entry.getTradeTimestamp()))); + joiner.add(QORTAL_EXCHANGE_LABEL); + + bufferedWriter.newLine(); + bufferedWriter.append(joiner.toString()); + } + + bufferedWriter.newLine(); + bufferedWriter.flush(); + } + + /** + * Create Ledger File Name + * + * Create a file name the includes timestamp and address. + * + * @param address the address + * + * @return the file name created + */ + public static String createLedgerFileName(String address) { + DateFormat dateFormatter = new SimpleDateFormat("yyyyMMddHHmmss"); + String fileName = "ledger-" + address + "-" + dateFormatter.format(new Date()); + return fileName; + } + + /** + * Collect Ledger Entries + * + * @param publicKey the public key for the ledger entries, buy and sell + * @param repository the data repository + * @param minimumFinalHeight the minimum block height for entries to be collected + * @param entries the ledger entries to add to + * @param codeHash code hash for the entry blockchain + * @param acct the ACCT for the entry blockchain + * @param isBuy true collecting entries for a buy, otherwise false + * + * @throws DataException + */ + public static void collectLedgerEntries( + byte[] publicKey, + Repository repository, + Integer minimumFinalHeight, + List entries, + byte[] codeHash, + ACCT acct, + boolean isBuy) throws DataException { + + // get all the final AT states for the code hash (foreign coin) + List atStates + = repository.getATRepository().getMatchingFinalATStates( + codeHash, + isBuy ? publicKey : null, + !isBuy ? publicKey : null, + Boolean.TRUE, acct.getModeByteOffset(), + (long) AcctMode.REDEEMED.value, + minimumFinalHeight, + null, null, false + ); + + String foreignBlockchainCurrencyCode = acct.getBlockchain().getCurrencyCode(); + + // for each trade, build ledger entry, collect ledger entry + for (ATStateData atState : atStates) { + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState); + + // We also need block timestamp for use as trade timestamp + long localTimestamp = repository.getBlockRepository().getTimestampFromHeight(atState.getHeight()); + + if (localTimestamp == 0) { + // Try the archive + localTimestamp = repository.getBlockArchiveRepository().getTimestampFromHeight(atState.getHeight()); + } + + CrossChainTradeLedgerEntry ledgerEntry + = new CrossChainTradeLedgerEntry( + isBuy ? QORT_CURRENCY_CODE : foreignBlockchainCurrencyCode, + isBuy ? foreignBlockchainCurrencyCode : QORT_CURRENCY_CODE, + isBuy ? crossChainTradeData.qortAmount : crossChainTradeData.expectedForeignAmount, + 0, + foreignBlockchainCurrencyCode, + isBuy ? crossChainTradeData.expectedForeignAmount : crossChainTradeData.qortAmount, + localTimestamp); + + entries.add(ledgerEntry); + } + } } \ No newline at end of file diff --git a/src/main/java/org/qortal/api/websocket/DataMonitorSocket.java b/src/main/java/org/qortal/api/websocket/DataMonitorSocket.java new file mode 100644 index 00000000..a93bf2ed --- /dev/null +++ b/src/main/java/org/qortal/api/websocket/DataMonitorSocket.java @@ -0,0 +1,102 @@ +package org.qortal.api.websocket; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketException; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.qortal.api.ApiError; +import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.DataMonitorInfo; +import org.qortal.event.DataMonitorEvent; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.Listener; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.Base58; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.List; + +@WebSocket +@SuppressWarnings("serial") +public class DataMonitorSocket extends ApiWebSocket implements Listener { + + private static final Logger LOGGER = LogManager.getLogger(DataMonitorSocket.class); + + @Override + public void configure(WebSocketServletFactory factory) { + LOGGER.info("configure"); + + factory.register(DataMonitorSocket.class); + + EventBus.INSTANCE.addListener(this); + } + + @Override + public void listen(Event event) { + if (!(event instanceof DataMonitorEvent)) + return; + + DataMonitorEvent dataMonitorEvent = (DataMonitorEvent) event; + + for (Session session : getSessions()) + sendDataEventSummary(session, buildInfo(dataMonitorEvent)); + } + + private DataMonitorInfo buildInfo(DataMonitorEvent dataMonitorEvent) { + + return new DataMonitorInfo( + dataMonitorEvent.getTimestamp(), + dataMonitorEvent.getIdentifier(), + dataMonitorEvent.getName(), + dataMonitorEvent.getService(), + dataMonitorEvent.getDescription(), + dataMonitorEvent.getTransactionTimestamp(), + dataMonitorEvent.getLatestPutTimestamp() + ); + } + + @OnWebSocketConnect + @Override + public void onWebSocketConnect(Session session) { + super.onWebSocketConnect(session); + } + + @OnWebSocketClose + @Override + public void onWebSocketClose(Session session, int statusCode, String reason) { + super.onWebSocketClose(session, statusCode, reason); + } + + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* We ignore errors for now, but method here to silence log spam */ + } + + @OnWebSocketMessage + public void onWebSocketMessage(Session session, String message) { + LOGGER.info("onWebSocketMessage: message = " + message); + } + + private void sendDataEventSummary(Session session, DataMonitorInfo dataMonitorInfo) { + StringWriter stringWriter = new StringWriter(); + + try { + marshall(stringWriter, dataMonitorInfo); + + session.getRemote().sendStringByFuture(stringWriter.toString()); + } catch (IOException | WebSocketException e) { + // No output this time + } + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 78a9ee86..6d7e0e23 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -439,7 +439,15 @@ public class ArbitraryDataReader { // Ensure the complete hash matches the joined chunks if (!Arrays.equals(arbitraryDataFile.digest(), transactionData.getData())) { // Delete the invalid file - arbitraryDataFile.delete(); + LOGGER.info("Deleting invalid file: path = " + arbitraryDataFile.getFilePath()); + + if( arbitraryDataFile.delete() ) { + LOGGER.info("Deleted invalid file successfully: path = " + arbitraryDataFile.getFilePath()); + } + else { + LOGGER.warn("Could not delete invalid file: path = " + arbitraryDataFile.getFilePath()); + } + throw new DataException("Unable to validate complete file hash"); } } diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 21cbddc4..67e6dd43 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -39,6 +39,7 @@ import org.qortal.transform.block.BlockTransformer; import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.utils.Amounts; import org.qortal.utils.Base58; +import org.qortal.utils.Groups; import org.qortal.utils.NTP; import java.io.ByteArrayOutputStream; @@ -150,7 +151,7 @@ public class Block { final BlockChain blockChain = BlockChain.getInstance(); - ExpandedAccount(Repository repository, RewardShareData rewardShareData) throws DataException { + ExpandedAccount(Repository repository, RewardShareData rewardShareData, int blockHeight) throws DataException { this.rewardShareData = rewardShareData; this.sharePercent = this.rewardShareData.getSharePercent(); @@ -159,7 +160,12 @@ public class Block { this.isMinterFounder = Account.isFounder(mintingAccountData.getFlags()); this.isRecipientAlsoMinter = this.rewardShareData.getRecipient().equals(this.mintingAccount.getAddress()); - this.isMinterMember = repository.getGroupRepository().memberExists(BlockChain.getInstance().getMintingGroupId(), this.mintingAccount.getAddress()); + this.isMinterMember + = Groups.memberExistsInAnyGroup( + repository.getGroupRepository(), + Groups.getGroupIdsToMint(BlockChain.getInstance(), blockHeight), + this.mintingAccount.getAddress() + ); if (this.isRecipientAlsoMinter) { // Self-share: minter is also recipient @@ -435,9 +441,9 @@ public class Block { if (height >= BlockChain.getInstance().getGroupMemberCheckHeight()) { onlineAccounts.removeIf(a -> { try { - int groupId = BlockChain.getInstance().getMintingGroupId(); + List groupIdsToMint = Groups.getGroupIdsToMint(BlockChain.getInstance(), height); String address = Account.getRewardShareMintingAddress(repository, a.getPublicKey()); - boolean isMinterGroupMember = repository.getGroupRepository().memberExists(groupId, address); + boolean isMinterGroupMember = Groups.memberExistsInAnyGroup(repository.getGroupRepository(), groupIdsToMint, address); return !isMinterGroupMember; } catch (DataException e) { // Something went wrong, so remove the account @@ -753,7 +759,7 @@ public class Block { List expandedAccounts = new ArrayList<>(); for (RewardShareData rewardShare : this.cachedOnlineRewardShares) { - expandedAccounts.add(new ExpandedAccount(repository, rewardShare)); + expandedAccounts.add(new ExpandedAccount(repository, rewardShare, this.blockData.getHeight())); } this.cachedExpandedAccounts = expandedAccounts; @@ -2485,11 +2491,10 @@ public class Block { try (final Repository repository = RepositoryManager.getRepository()) { GroupRepository groupRepository = repository.getGroupRepository(); + List mintingGroupIds = Groups.getGroupIdsToMint(BlockChain.getInstance(), this.blockData.getHeight()); + // all minter admins - List minterAdmins - = groupRepository.getGroupAdmins(BlockChain.getInstance().getMintingGroupId()).stream() - .map(GroupAdminData::getAdmin) - .collect(Collectors.toList()); + List minterAdmins = Groups.getAllAdmins(groupRepository, mintingGroupIds); // all minter admins that are online List onlineMinterAdminAccounts diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 1468fbc3..bce09aed 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -212,7 +212,13 @@ public class BlockChain { private int minAccountLevelToRewardShare; private int maxRewardSharesPerFounderMintingAccount; private int founderEffectiveMintingLevel; - private int mintingGroupId; + + public static class IdsForHeight { + public int height; + public List ids; + } + + private List mintingGroupIds; /** Minimum time to retain online account signatures (ms) for block validity checks. */ private long onlineAccountSignaturesMinLifetime; @@ -544,8 +550,8 @@ public class BlockChain { return this.onlineAccountSignaturesMaxLifetime; } - public int getMintingGroupId() { - return this.mintingGroupId; + public List getMintingGroupIds() { + return mintingGroupIds; } public CiyamAtSettings getCiyamAtSettings() { diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 5697f1ae..4f885e7e 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -427,6 +427,12 @@ public class Controller extends Thread { LOGGER.info("Db Cache Disabled"); } + LOGGER.info("Arbitrary Indexing Starting ..."); + ArbitraryIndexUtils.startCaching( + Settings.getInstance().getArbitraryIndexingPriority(), + Settings.getInstance().getArbitraryIndexingFrequency() + ); + if( Settings.getInstance().isBalanceRecorderEnabled() ) { Optional recorder = HSQLDBBalanceRecorder.getInstance(); @@ -554,6 +560,16 @@ public class Controller extends Thread { ArbitraryDataStorageManager.getInstance().start(); ArbitraryDataRenderManager.getInstance().start(); + // start rebuild arbitrary resource cache timer task + if( Settings.getInstance().isRebuildArbitraryResourceCacheTaskEnabled() ) { + new Timer().schedule( + new RebuildArbitraryResourceCacheTask(), + Settings.getInstance().getRebuildArbitraryResourceCacheTaskDelay() * RebuildArbitraryResourceCacheTask.MILLIS_IN_MINUTE, + Settings.getInstance().getRebuildArbitraryResourceCacheTaskPeriod() * RebuildArbitraryResourceCacheTask.MILLIS_IN_HOUR + ); + } + + LOGGER.info("Starting online accounts manager"); OnlineAccountsManager.getInstance().start(); diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 332bf867..bbca4c7b 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -25,6 +25,7 @@ import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.utils.Base58; +import org.qortal.utils.Groups; import org.qortal.utils.NTP; import org.qortal.utils.NamedThreadFactory; @@ -225,11 +226,14 @@ public class OnlineAccountsManager { Set onlineAccountsToAdd = new HashSet<>(); Set onlineAccountsToRemove = new HashSet<>(); try (final Repository repository = RepositoryManager.getRepository()) { + + int blockHeight = repository.getBlockRepository().getBlockchainHeight(); + List mintingGroupMemberAddresses - = repository.getGroupRepository() - .getGroupMembers(BlockChain.getInstance().getMintingGroupId()).stream() - .map(GroupMemberData::getMember) - .collect(Collectors.toList()); + = Groups.getAllMembers( + repository.getGroupRepository(), + Groups.getGroupIdsToMint(BlockChain.getInstance(), blockHeight) + ); for (OnlineAccountData onlineAccountData : this.onlineAccountsImportQueue) { if (isStopping) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java index d6b9303f..9accd9c7 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java @@ -2,22 +2,30 @@ package org.qortal.controller.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.api.resource.TransactionsResource; import org.qortal.controller.Controller; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.event.DataMonitorEvent; +import org.qortal.event.EventBus; import org.qortal.gui.SplashFrame; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.transaction.ArbitraryTransaction; -import org.qortal.transaction.Transaction; import org.qortal.utils.Base58; +import java.text.NumberFormat; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; public class ArbitraryDataCacheManager extends Thread { @@ -29,6 +37,11 @@ public class ArbitraryDataCacheManager extends Thread { /** Queue of arbitrary transactions that require cache updates */ private final List updateQueue = Collections.synchronizedList(new ArrayList<>()); + private static final NumberFormat FORMATTER = NumberFormat.getNumberInstance(); + + static { + FORMATTER.setGroupingUsed(true); + } public static synchronized ArbitraryDataCacheManager getInstance() { if (instance == null) { @@ -45,17 +58,22 @@ public class ArbitraryDataCacheManager extends Thread { try { while (!Controller.isStopping()) { - Thread.sleep(500L); + try { + Thread.sleep(500L); - // Process queue - processResourceQueue(); + // Process queue + processResourceQueue(); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + Thread.sleep(600_000L); // wait 10 minutes to continue + } } - } catch (InterruptedException e) { - // Fall through to exit thread - } - // Clear queue before terminating thread - processResourceQueue(); + // Clear queue before terminating thread + processResourceQueue(); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } } public void shutdown() { @@ -85,14 +103,25 @@ public class ArbitraryDataCacheManager extends Thread { // Update arbitrary resource caches try { ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); - arbitraryTransaction.updateArbitraryResourceCache(repository); - arbitraryTransaction.updateArbitraryMetadataCache(repository); + arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, new HashSet<>(0), new HashMap<>(0)); repository.saveChanges(); // Update status as separate commit, as this is more prone to failure arbitraryTransaction.updateArbitraryResourceStatus(repository); repository.saveChanges(); + EventBus.INSTANCE.notify( + new DataMonitorEvent( + System.currentTimeMillis(), + transactionData.getIdentifier(), + transactionData.getName(), + transactionData.getService().name(), + "updated resource cache and status, queue", + transactionData.getTimestamp(), + transactionData.getTimestamp() + ) + ); + LOGGER.debug(() -> String.format("Finished processing transaction %.8s in arbitrary resource queue...", Base58.encode(transactionData.getSignature()))); } catch (DataException e) { @@ -103,6 +132,9 @@ public class ArbitraryDataCacheManager extends Thread { } catch (DataException e) { LOGGER.error("Repository issue while processing arbitrary resource cache updates", e); } + catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } } public void addToUpdateQueue(ArbitraryTransactionData transactionData) { @@ -148,34 +180,66 @@ public class ArbitraryDataCacheManager extends Thread { LOGGER.info("Building arbitrary resources cache..."); SplashFrame.getInstance().updateStatus("Building QDN cache - please wait..."); - final int batchSize = 100; + final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize(); int offset = 0; + List allArbitraryTransactionsInDescendingOrder + = repository.getArbitraryRepository().getLatestArbitraryTransactions(); + + LOGGER.info("arbitrary transactions: count = " + allArbitraryTransactionsInDescendingOrder.size()); + + List resources = repository.getArbitraryRepository().getArbitraryResources(null, null, true); + + Map resourceByWrapper = new HashMap<>(resources.size()); + for( ArbitraryResourceData resource : resources ) { + resourceByWrapper.put( + new ArbitraryTransactionDataHashWrapper(resource.service.value, resource.name, resource.identifier), + resource + ); + } + + LOGGER.info("arbitrary resources: count = " + resourceByWrapper.size()); + + Set latestTransactionsWrapped = new HashSet<>(allArbitraryTransactionsInDescendingOrder.size()); + // Loop through all ARBITRARY transactions, and determine latest state while (!Controller.isStopping()) { - LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1); + LOGGER.info( + "Fetching arbitrary transactions {} - {} / {} Total", + FORMATTER.format(offset), + FORMATTER.format(offset+batchSize-1), + FORMATTER.format(allArbitraryTransactionsInDescendingOrder.size()) + ); - List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false); - if (signatures.isEmpty()) { + List transactionsToProcess + = allArbitraryTransactionsInDescendingOrder.stream() + .skip(offset) + .limit(batchSize) + .collect(Collectors.toList()); + + if (transactionsToProcess.isEmpty()) { // Complete break; } - // Expand signatures to transactions - for (byte[] signature : signatures) { - ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository - .getTransactionRepository().fromSignature(signature); + try { + for( ArbitraryTransactionData transactionData : transactionsToProcess) { + if (transactionData.getService() == null) { + // Unsupported service - ignore this resource + continue; + } - if (transactionData.getService() == null) { - // Unsupported service - ignore this resource - continue; + latestTransactionsWrapped.add(new ArbitraryTransactionDataHashWrapper(transactionData)); + + // Update arbitrary resource caches + ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); + arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, latestTransactionsWrapped, resourceByWrapper); } - - // Update arbitrary resource caches - ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); - arbitraryTransaction.updateArbitraryResourceCache(repository); - arbitraryTransaction.updateArbitraryMetadataCache(repository); repository.saveChanges(); + } catch (DataException e) { + repository.discardChanges(); + + LOGGER.error(e.getMessage(), e); } offset += batchSize; } @@ -193,6 +257,11 @@ public class ArbitraryDataCacheManager extends Thread { repository.discardChanges(); throw new DataException("Build of arbitrary resources cache failed."); } + catch (Exception e) { + LOGGER.error(e.getMessage(), e); + + return false; + } } private boolean refreshArbitraryStatuses(Repository repository) throws DataException { @@ -200,27 +269,48 @@ public class ArbitraryDataCacheManager extends Thread { LOGGER.info("Refreshing arbitrary resource statuses for locally hosted transactions..."); SplashFrame.getInstance().updateStatus("Refreshing statuses - please wait..."); - final int batchSize = 100; + final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize(); int offset = 0; + List allHostedTransactions + = ArbitraryDataStorageManager.getInstance() + .listAllHostedTransactions(repository, null, null); + // Loop through all ARBITRARY transactions, and determine latest state while (!Controller.isStopping()) { - LOGGER.info("Fetching hosted transactions {} - {}", offset, offset+batchSize-1); + LOGGER.info( + "Fetching hosted transactions {} - {} / {} Total", + FORMATTER.format(offset), + FORMATTER.format(offset+batchSize-1), + FORMATTER.format(allHostedTransactions.size()) + ); + + List hostedTransactions + = allHostedTransactions.stream() + .skip(offset) + .limit(batchSize) + .collect(Collectors.toList()); - List hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, batchSize, offset); if (hostedTransactions.isEmpty()) { // Complete break; } - // Loop through hosted transactions - for (ArbitraryTransactionData transactionData : hostedTransactions) { + try { + // Loop through hosted transactions + for (ArbitraryTransactionData transactionData : hostedTransactions) { - // Determine status and update cache - ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); - arbitraryTransaction.updateArbitraryResourceStatus(repository); + // Determine status and update cache + ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); + arbitraryTransaction.updateArbitraryResourceStatus(repository); + } repository.saveChanges(); + } catch (DataException e) { + repository.discardChanges(); + + LOGGER.error(e.getMessage(), e); } + offset += batchSize; } @@ -234,6 +324,11 @@ public class ArbitraryDataCacheManager extends Thread { repository.discardChanges(); throw new DataException("Refresh of arbitrary resource statuses failed."); } + catch (Exception e) { + LOGGER.error(e.getMessage(), e); + + return false; + } } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index aa29a7b8..ce4dd565 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -2,9 +2,10 @@ package org.qortal.controller.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; +import org.qortal.event.DataMonitorEvent; +import org.qortal.event.EventBus; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -21,8 +22,12 @@ import java.nio.file.Paths; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD; @@ -77,6 +82,19 @@ public class ArbitraryDataCleanupManager extends Thread { final int limit = 100; int offset = 0; + List allArbitraryTransactionsInDescendingOrder; + + try (final Repository repository = RepositoryManager.getRepository()) { + allArbitraryTransactionsInDescendingOrder + = repository.getArbitraryRepository() + .getLatestArbitraryTransactions(); + } catch( Exception e) { + LOGGER.error(e.getMessage(), e); + allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0); + } + + Set processedTransactions = new HashSet<>(); + try { while (!isStopping) { Thread.sleep(30000); @@ -107,27 +125,31 @@ public class ArbitraryDataCleanupManager extends Thread { // Any arbitrary transactions we want to fetch data for? try (final Repository repository = RepositoryManager.getRepository()) { - List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true); - // LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit); + List transactions = allArbitraryTransactionsInDescendingOrder.stream().skip(offset).limit(limit).collect(Collectors.toList()); if (isStopping) { return; } - if (signatures == null || signatures.isEmpty()) { + if (transactions == null || transactions.isEmpty()) { offset = 0; - continue; + allArbitraryTransactionsInDescendingOrder + = repository.getArbitraryRepository() + .getLatestArbitraryTransactions(); + transactions = allArbitraryTransactionsInDescendingOrder.stream().limit(limit).collect(Collectors.toList()); + processedTransactions.clear(); } + offset += limit; now = NTP.getTime(); // Loop through the signatures in this batch - for (int i=0; i moreRecentPutTransaction + = processedTransactions.stream() + .filter(data -> data.equals(arbitraryTransactionData)) + .findAny(); + + if( moreRecentPutTransaction.isPresent() ) { + EventBus.INSTANCE.notify( + new DataMonitorEvent( + System.currentTimeMillis(), + arbitraryTransactionData.getIdentifier(), + arbitraryTransactionData.getName(), + arbitraryTransactionData.getService().name(), + "deleting data due to replacement", + arbitraryTransactionData.getTimestamp(), + moreRecentPutTransaction.get().getTimestamp() + ) + ); + } + else { + LOGGER.warn("Something went wrong with the most recent put transaction determination!"); + } + continue; } @@ -199,7 +255,21 @@ public class ArbitraryDataCleanupManager extends Thread { LOGGER.debug(String.format("Transaction %s has complete file and all chunks", Base58.encode(arbitraryTransactionData.getSignature()))); - ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT); + boolean wasDeleted = ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT); + + if( wasDeleted ) { + EventBus.INSTANCE.notify( + new DataMonitorEvent( + System.currentTimeMillis(), + arbitraryTransactionData.getIdentifier(), + arbitraryTransactionData.getName(), + arbitraryTransactionData.getService().name(), + "deleting file, retaining chunks", + arbitraryTransactionData.getTimestamp(), + arbitraryTransactionData.getTimestamp() + ) + ); + } continue; } @@ -237,17 +307,6 @@ public class ArbitraryDataCleanupManager extends Thread { this.storageLimitReached(repository); } - // Delete random data associated with name if we're over our storage limit for this name - // Use the DELETION_THRESHOLD, for the same reasons as above - for (String followedName : ListUtils.followedNames()) { - if (isStopping) { - return; - } - if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) { - this.storageLimitReachedForName(repository, followedName); - } - } - } catch (DataException e) { LOGGER.error("Repository issue when cleaning up arbitrary transaction data", e); } @@ -326,25 +385,6 @@ public class ArbitraryDataCleanupManager extends Thread { // FUTURE: consider reducing the expiry time of the reader cache } - public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException { - // We think that the storage limit has been reached for supplied name - but we should double check - if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, DELETION_THRESHOLD)) { - // We have space available for this name, so don't delete anything - return; - } - - // Delete a batch of random chunks associated with this name - // This reduces the chance of too many nodes deleting the same chunk - // when they reach their storage limit - Path dataPath = Paths.get(Settings.getInstance().getDataPath()); - for (int i=0; i allArbitraryTransactionsInDescendingOrder; + + try (final Repository repository = RepositoryManager.getRepository()) { + + if( name == null ) { + allArbitraryTransactionsInDescendingOrder + = repository.getArbitraryRepository() + .getLatestArbitraryTransactions(); + } + else { + allArbitraryTransactionsInDescendingOrder + = repository.getArbitraryRepository() + .getLatestArbitraryTransactionsByName(name); + } + } catch( Exception e) { + LOGGER.error(e.getMessage(), e); + allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0); + } + + // collect processed transactions in a set to ensure outdated data transactions do not get fetched + Set processedTransactions = new HashSet<>(); + while (!isStopping) { Thread.sleep(1000L); // Any arbitrary transactions we want to fetch data for? try (final Repository repository = RepositoryManager.getRepository()) { - List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, name, null, ConfirmationStatus.BOTH, limit, offset, true); - // LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit); + List signatures = processTransactionsForSignatures(limit, offset, allArbitraryTransactionsInDescendingOrder, processedTransactions); + if (signatures == null || signatures.isEmpty()) { offset = 0; break; @@ -223,14 +248,38 @@ public class ArbitraryDataManager extends Thread { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData(); // Skip transactions that we don't need to proactively store data for - if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) { + ArbitraryDataExamination arbitraryDataExamination = storageManager.shouldPreFetchData(repository, arbitraryTransactionData); + if (!arbitraryDataExamination.isPass()) { iterator.remove(); + + EventBus.INSTANCE.notify( + new DataMonitorEvent( + System.currentTimeMillis(), + arbitraryTransactionData.getIdentifier(), + arbitraryTransactionData.getName(), + arbitraryTransactionData.getService().name(), + arbitraryDataExamination.getNotes(), + arbitraryTransactionData.getTimestamp(), + arbitraryTransactionData.getTimestamp() + ) + ); continue; } // Remove transactions that we already have local data for if (hasLocalData(arbitraryTransaction)) { iterator.remove(); + EventBus.INSTANCE.notify( + new DataMonitorEvent( + System.currentTimeMillis(), + arbitraryTransactionData.getIdentifier(), + arbitraryTransactionData.getName(), + arbitraryTransactionData.getService().name(), + "already have local data, skipping", + arbitraryTransactionData.getTimestamp(), + arbitraryTransactionData.getTimestamp() + ) + ); } } @@ -248,8 +297,21 @@ public class ArbitraryDataManager extends Thread { // Check to see if we have had a more recent PUT ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); - boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData); - if (hasMoreRecentPutTransaction) { + + Optional moreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData); + + if (moreRecentPutTransaction.isPresent()) { + EventBus.INSTANCE.notify( + new DataMonitorEvent( + System.currentTimeMillis(), + arbitraryTransactionData.getIdentifier(), + arbitraryTransactionData.getName(), + arbitraryTransactionData.getService().name(), + "not fetching old data", + arbitraryTransactionData.getTimestamp(), + moreRecentPutTransaction.get().getTimestamp() + ) + ); // There is a more recent PUT transaction than the one we are currently processing. // When a PUT is issued, it replaces any layers that would have been there before. // Therefore any data relating to this older transaction is no longer needed and we @@ -257,10 +319,34 @@ public class ArbitraryDataManager extends Thread { continue; } + EventBus.INSTANCE.notify( + new DataMonitorEvent( + System.currentTimeMillis(), + arbitraryTransactionData.getIdentifier(), + arbitraryTransactionData.getName(), + arbitraryTransactionData.getService().name(), + "fetching data", + arbitraryTransactionData.getTimestamp(), + arbitraryTransactionData.getTimestamp() + ) + ); + // Ask our connected peers if they have files for this signature // This process automatically then fetches the files themselves if a peer is found fetchData(arbitraryTransactionData); + EventBus.INSTANCE.notify( + new DataMonitorEvent( + System.currentTimeMillis(), + arbitraryTransactionData.getIdentifier(), + arbitraryTransactionData.getName(), + arbitraryTransactionData.getService().name(), + "fetched data", + arbitraryTransactionData.getTimestamp(), + arbitraryTransactionData.getTimestamp() + ) + ); + } catch (DataException e) { LOGGER.error("Repository issue when fetching arbitrary transaction data", e); } @@ -274,6 +360,20 @@ public class ArbitraryDataManager extends Thread { final int limit = 100; int offset = 0; + List allArbitraryTransactionsInDescendingOrder; + + try (final Repository repository = RepositoryManager.getRepository()) { + allArbitraryTransactionsInDescendingOrder + = repository.getArbitraryRepository() + .getLatestArbitraryTransactions(); + } catch( Exception e) { + LOGGER.error(e.getMessage(), e); + allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0); + } + + // collect processed transactions in a set to ensure outdated data transactions do not get fetched + Set processedTransactions = new HashSet<>(); + while (!isStopping) { final int minSeconds = 3; final int maxSeconds = 10; @@ -282,8 +382,8 @@ public class ArbitraryDataManager extends Thread { // Any arbitrary transactions we want to fetch data for? try (final Repository repository = RepositoryManager.getRepository()) { - List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true); - // LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit); + List signatures = processTransactionsForSignatures(limit, offset, allArbitraryTransactionsInDescendingOrder, processedTransactions); + if (signatures == null || signatures.isEmpty()) { offset = 0; break; @@ -328,26 +428,74 @@ public class ArbitraryDataManager extends Thread { continue; } - // Check to see if we have had a more recent PUT + // No longer need to see if we have had a more recent PUT since we compared the transactions to process + // to the transactions previously processed, so we can fetch the transactiondata, notify the event bus, + // fetch the metadata and notify the event bus again ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); - boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData); - if (hasMoreRecentPutTransaction) { - // There is a more recent PUT transaction than the one we are currently processing. - // When a PUT is issued, it replaces any layers that would have been there before. - // Therefore any data relating to this older transaction is no longer needed and we - // shouldn't fetch it from the network. - continue; - } // Ask our connected peers if they have metadata for this signature fetchMetadata(arbitraryTransactionData); + EventBus.INSTANCE.notify( + new DataMonitorEvent( + System.currentTimeMillis(), + arbitraryTransactionData.getIdentifier(), + arbitraryTransactionData.getName(), + arbitraryTransactionData.getService().name(), + "fetched metadata", + arbitraryTransactionData.getTimestamp(), + arbitraryTransactionData.getTimestamp() + ) + ); } catch (DataException e) { LOGGER.error("Repository issue when fetching arbitrary transaction data", e); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } } + private static List processTransactionsForSignatures( + int limit, + int offset, + List transactionsInDescendingOrder, + Set processedTransactions) { + // these transactions are in descending order, latest transactions come first + List transactions + = transactionsInDescendingOrder.stream() + .skip(offset) + .limit(limit) + .collect(Collectors.toList()); + + // wrap the transactions, so they can be used for hashing and comparing + // Class ArbitraryTransactionDataHashWrapper supports hashCode() and equals(...) for this purpose + List wrappedTransactions + = transactions.stream() + .map(transaction -> new ArbitraryTransactionDataHashWrapper(transaction)) + .collect(Collectors.toList()); + + // create a set of wrappers and populate it first to last, so that all outdated transactions get rejected + Set transactionsToProcess = new HashSet<>(wrappedTransactions.size()); + for(ArbitraryTransactionDataHashWrapper wrappedTransaction : wrappedTransactions) { + transactionsToProcess.add(wrappedTransaction); + } + + // remove the matches for previously processed transactions, + // because these transactions have had updates that have already been processed + transactionsToProcess.removeAll(processedTransactions); + + // add to processed transactions to compare and remove matches from future processing iterations + processedTransactions.addAll(transactionsToProcess); + + List signatures + = transactionsToProcess.stream() + .map(transactionToProcess -> transactionToProcess.getData() + .getSignature()) + .collect(Collectors.toList()); + + return signatures; + } + private ArbitraryTransaction fetchTransaction(final Repository repository, byte[] signature) { try { TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index 62feb8bd..c54a1e12 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -155,31 +155,24 @@ public class ArbitraryDataStorageManager extends Thread { * @param arbitraryTransactionData - the transaction * @return boolean - whether to prefetch or not */ - public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { + public ArbitraryDataExamination shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { String name = arbitraryTransactionData.getName(); // Only fetch data associated with hashes, as we already have RAW_DATA if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) { - return false; + return new ArbitraryDataExamination(false, "Only fetch data associated with hashes"); } // Don't fetch anything more if we're (nearly) out of space // Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to // avoid a fetch/delete loop if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) { - return false; - } - - // Don't fetch anything if we're (nearly) out of space for this name - // Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to - // avoid a fetch/delete loop - if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) { - return false; + return new ArbitraryDataExamination(false,"Don't fetch anything more if we're (nearly) out of space"); } // Don't store data unless it's an allowed type (public/private) if (!this.isDataTypeAllowed(arbitraryTransactionData)) { - return false; + return new ArbitraryDataExamination(false, "Don't store data unless it's an allowed type (public/private)"); } // Handle transactions without names differently @@ -189,21 +182,21 @@ public class ArbitraryDataStorageManager extends Thread { // Never fetch data from blocked names, even if they are followed if (ListUtils.isNameBlocked(name)) { - return false; + return new ArbitraryDataExamination(false, "blocked name"); } switch (Settings.getInstance().getStoragePolicy()) { case FOLLOWED: case FOLLOWED_OR_VIEWED: - return ListUtils.isFollowingName(name); + return new ArbitraryDataExamination(ListUtils.isFollowingName(name), Settings.getInstance().getStoragePolicy().name()); case ALL: - return true; + return new ArbitraryDataExamination(true, Settings.getInstance().getStoragePolicy().name()); case NONE: case VIEWED: default: - return false; + return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name()); } } @@ -214,17 +207,17 @@ public class ArbitraryDataStorageManager extends Thread { * * @return boolean - whether the storage policy allows for unnamed data */ - private boolean shouldPreFetchDataWithoutName() { + private ArbitraryDataExamination shouldPreFetchDataWithoutName() { switch (Settings.getInstance().getStoragePolicy()) { case ALL: - return true; + return new ArbitraryDataExamination(true, "Fetching all data"); case NONE: case VIEWED: case FOLLOWED: case FOLLOWED_OR_VIEWED: default: - return false; + return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name()); } } @@ -484,51 +477,6 @@ public class ArbitraryDataStorageManager extends Thread { return true; } - public boolean isStorageSpaceAvailableForName(Repository repository, String name, double threshold) { - if (!this.isStorageSpaceAvailable(threshold)) { - // No storage space available at all, so no need to check this name - return false; - } - - if (Settings.getInstance().getStoragePolicy() == StoragePolicy.ALL) { - // Using storage policy ALL, so don't limit anything per name - return true; - } - - if (name == null) { - // This transaction doesn't have a name, so fall back to total space limitations - return true; - } - - int followedNamesCount = ListUtils.followedNamesCount(); - if (followedNamesCount == 0) { - // Not following any names, so we have space - return true; - } - - long totalSizeForName = 0; - long maxStoragePerName = this.storageCapacityPerName(threshold); - - // Fetch all hosted transactions - List hostedTransactions = this.listAllHostedTransactions(repository, null, null); - for (ArbitraryTransactionData transactionData : hostedTransactions) { - String transactionName = transactionData.getName(); - if (!Objects.equals(name, transactionName)) { - // Transaction relates to a different name - continue; - } - - totalSizeForName += transactionData.getSize(); - } - - // Have we reached the limit for this name? - if (totalSizeForName > maxStoragePerName) { - return false; - } - - return true; - } - public long storageCapacityPerName(double threshold) { int followedNamesCount = ListUtils.followedNamesCount(); if (followedNamesCount == 0) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryTransactionDataHashWrapper.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryTransactionDataHashWrapper.java new file mode 100644 index 00000000..9ff40771 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryTransactionDataHashWrapper.java @@ -0,0 +1,48 @@ +package org.qortal.controller.arbitrary; + +import org.qortal.arbitrary.misc.Service; +import org.qortal.data.transaction.ArbitraryTransactionData; + +import java.util.Objects; + +public class ArbitraryTransactionDataHashWrapper { + + private ArbitraryTransactionData data; + + private int service; + + private String name; + + private String identifier; + + public ArbitraryTransactionDataHashWrapper(ArbitraryTransactionData data) { + this.data = data; + + this.service = data.getService().value; + this.name = data.getName(); + this.identifier = data.getIdentifier(); + } + + public ArbitraryTransactionDataHashWrapper(int service, String name, String identifier) { + this.service = service; + this.name = name; + this.identifier = identifier; + } + + public ArbitraryTransactionData getData() { + return data; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ArbitraryTransactionDataHashWrapper that = (ArbitraryTransactionDataHashWrapper) o; + return service == that.service && name.equals(that.name) && Objects.equals(identifier, that.identifier); + } + + @Override + public int hashCode() { + return Objects.hash(service, name, identifier); + } +} diff --git a/src/main/java/org/qortal/controller/arbitrary/RebuildArbitraryResourceCacheTask.java b/src/main/java/org/qortal/controller/arbitrary/RebuildArbitraryResourceCacheTask.java new file mode 100644 index 00000000..d7472325 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/RebuildArbitraryResourceCacheTask.java @@ -0,0 +1,33 @@ +package org.qortal.controller.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; + +import java.util.TimerTask; + +public class RebuildArbitraryResourceCacheTask extends TimerTask { + + private static final Logger LOGGER = LogManager.getLogger(RebuildArbitraryResourceCacheTask.class); + + public static final long MILLIS_IN_HOUR = 60 * 60 * 1000; + + public static final long MILLIS_IN_MINUTE = 60 * 1000; + + private static final String REBUILD_ARBITRARY_RESOURCE_CACHE_TASK = "Rebuild Arbitrary Resource Cache Task"; + + @Override + public void run() { + + Thread.currentThread().setName(REBUILD_ARBITRARY_RESOURCE_CACHE_TASK); + + try (final Repository repository = RepositoryManager.getRepository()) { + ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, true); + } + catch( DataException e ) { + LOGGER.error(e.getMessage(), e); + } + } +} diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index a4f5a2af..d93fa65f 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -83,6 +83,7 @@ public abstract class Bitcoiny implements ForeignBlockchain { return this.bitcoinjContext; } + @Override public String getCurrencyCode() { return this.currencyCode; } diff --git a/src/main/java/org/qortal/crosschain/ForeignBlockchain.java b/src/main/java/org/qortal/crosschain/ForeignBlockchain.java index fe64ab83..c66f2719 100644 --- a/src/main/java/org/qortal/crosschain/ForeignBlockchain.java +++ b/src/main/java/org/qortal/crosschain/ForeignBlockchain.java @@ -2,6 +2,8 @@ package org.qortal.crosschain; public interface ForeignBlockchain { + public String getCurrencyCode(); + public boolean isValidAddress(String address); public boolean isValidWalletKey(String walletKey); diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java new file mode 100644 index 00000000..9b6bf415 --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java @@ -0,0 +1,34 @@ +package org.qortal.data.arbitrary; + +import org.qortal.arbitrary.misc.Service; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ArbitraryDataIndex { + + public String t; + public String n; + public int c; + public String l; + + public ArbitraryDataIndex() {} + + public ArbitraryDataIndex(String t, String n, int c, String l) { + this.t = t; + this.n = n; + this.c = c; + this.l = l; + } + + @Override + public String toString() { + return "ArbitraryDataIndex{" + + "t='" + t + '\'' + + ", n='" + n + '\'' + + ", c=" + c + + ", l='" + l + '\'' + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java new file mode 100644 index 00000000..d073c736 --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java @@ -0,0 +1,41 @@ +package org.qortal.data.arbitrary; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ArbitraryDataIndexDetail { + + public String issuer; + public int rank; + public String term; + public String name; + public int category; + public String link; + public String indexIdentifer; + + public ArbitraryDataIndexDetail() {} + + public ArbitraryDataIndexDetail(String issuer, int rank, ArbitraryDataIndex index, String indexIdentifer) { + this.issuer = issuer; + this.rank = rank; + this.term = index.t; + this.name = index.n; + this.category = index.c; + this.link = index.l; + this.indexIdentifer = indexIdentifer; + } + + @Override + public String toString() { + return "ArbitraryDataIndexDetail{" + + "issuer='" + issuer + '\'' + + ", rank=" + rank + + ", term='" + term + '\'' + + ", name='" + name + '\'' + + ", category=" + category + + ", link='" + link + '\'' + + ", indexIdentifer='" + indexIdentifer + '\'' + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java new file mode 100644 index 00000000..46a661e5 --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java @@ -0,0 +1,38 @@ +package org.qortal.data.arbitrary; + +import org.qortal.arbitrary.misc.Service; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ArbitraryDataIndexScoreKey { + + public String name; + public int category; + public String link; + + public ArbitraryDataIndexScoreKey() {} + + public ArbitraryDataIndexScoreKey(String name, int category, String link) { + this.name = name; + this.category = category; + this.link = link; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ArbitraryDataIndexScoreKey that = (ArbitraryDataIndexScoreKey) o; + return category == that.category && Objects.equals(name, that.name) && Objects.equals(link, that.link); + } + + @Override + public int hashCode() { + return Objects.hash(name, category, link); + } + + +} diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java new file mode 100644 index 00000000..1888a4a5 --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java @@ -0,0 +1,38 @@ +package org.qortal.data.arbitrary; + +import org.qortal.arbitrary.misc.Service; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ArbitraryDataIndexScorecard { + + public double score; + public String name; + public int category; + public String link; + + public ArbitraryDataIndexScorecard() {} + + public ArbitraryDataIndexScorecard(double score, String name, int category, String link) { + this.score = score; + this.name = name; + this.category = category; + this.link = link; + } + + public double getScore() { + return score; + } + + @Override + public String toString() { + return "ArbitraryDataIndexScorecard{" + + "score=" + score + + ", name='" + name + '\'' + + ", category=" + category + + ", link='" + link + '\'' + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/arbitrary/DataMonitorInfo.java b/src/main/java/org/qortal/data/arbitrary/DataMonitorInfo.java new file mode 100644 index 00000000..5ee76c29 --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/DataMonitorInfo.java @@ -0,0 +1,57 @@ +package org.qortal.data.arbitrary; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class DataMonitorInfo { + private long timestamp; + private String identifier; + private String name; + private String service; + private String description; + private long transactionTimestamp; + private long latestPutTimestamp; + + public DataMonitorInfo() { + } + + public DataMonitorInfo(long timestamp, String identifier, String name, String service, String description, long transactionTimestamp, long latestPutTimestamp) { + + this.timestamp = timestamp; + this.identifier = identifier; + this.name = name; + this.service = service; + this.description = description; + this.transactionTimestamp = transactionTimestamp; + this.latestPutTimestamp = latestPutTimestamp; + } + + public long getTimestamp() { + return timestamp; + } + + public String getIdentifier() { + return identifier; + } + + public String getName() { + return name; + } + + public String getService() { + return service; + } + + public String getDescription() { + return description; + } + + public long getTransactionTimestamp() { + return transactionTimestamp; + } + + public long getLatestPutTimestamp() { + return latestPutTimestamp; + } +} diff --git a/src/main/java/org/qortal/data/arbitrary/IndexCache.java b/src/main/java/org/qortal/data/arbitrary/IndexCache.java new file mode 100644 index 00000000..dd5c12ab --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/IndexCache.java @@ -0,0 +1,23 @@ +package org.qortal.data.arbitrary; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +public class IndexCache { + + public static final IndexCache SINGLETON = new IndexCache(); + private ConcurrentHashMap> indicesByTerm = new ConcurrentHashMap<>(); + private ConcurrentHashMap> indicesByIssuer = new ConcurrentHashMap<>(); + + public static IndexCache getInstance() { + return SINGLETON; + } + + public ConcurrentHashMap> getIndicesByTerm() { + return indicesByTerm; + } + + public ConcurrentHashMap> getIndicesByIssuer() { + return indicesByIssuer; + } +} diff --git a/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java b/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java index f3828de8..81cfaa68 100644 --- a/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java +++ b/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java @@ -200,4 +200,26 @@ public class ArbitraryTransactionData extends TransactionData { return this.payments; } + @Override + public String toString() { + return "ArbitraryTransactionData{" + + "version=" + version + + ", service=" + service + + ", nonce=" + nonce + + ", size=" + size + + ", name='" + name + '\'' + + ", identifier='" + identifier + '\'' + + ", method=" + method + + ", compression=" + compression + + ", dataType=" + dataType + + ", type=" + type + + ", timestamp=" + timestamp + + ", fee=" + fee + + ", txGroupId=" + txGroupId + + ", blockHeight=" + blockHeight + + ", blockSequence=" + blockSequence + + ", approvalStatus=" + approvalStatus + + ", approvalHeight=" + approvalHeight + + '}'; + } } diff --git a/src/main/java/org/qortal/event/DataMonitorEvent.java b/src/main/java/org/qortal/event/DataMonitorEvent.java new file mode 100644 index 00000000..c62d9acf --- /dev/null +++ b/src/main/java/org/qortal/event/DataMonitorEvent.java @@ -0,0 +1,57 @@ +package org.qortal.event; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class DataMonitorEvent implements Event{ + private long timestamp; + private String identifier; + private String name; + private String service; + private String description; + private long transactionTimestamp; + private long latestPutTimestamp; + + public DataMonitorEvent() { + } + + public DataMonitorEvent(long timestamp, String identifier, String name, String service, String description, long transactionTimestamp, long latestPutTimestamp) { + + this.timestamp = timestamp; + this.identifier = identifier; + this.name = name; + this.service = service; + this.description = description; + this.transactionTimestamp = transactionTimestamp; + this.latestPutTimestamp = latestPutTimestamp; + } + + public long getTimestamp() { + return timestamp; + } + + public String getIdentifier() { + return identifier; + } + + public String getName() { + return name; + } + + public String getService() { + return service; + } + + public String getDescription() { + return description; + } + + public long getTransactionTimestamp() { + return transactionTimestamp; + } + + public long getLatestPutTimestamp() { + return latestPutTimestamp; + } +} diff --git a/src/main/java/org/qortal/group/Group.java b/src/main/java/org/qortal/group/Group.java index 765b86de..a6b4f3a6 100644 --- a/src/main/java/org/qortal/group/Group.java +++ b/src/main/java/org/qortal/group/Group.java @@ -674,8 +674,8 @@ public class Group { public void uninvite(GroupInviteTransactionData groupInviteTransactionData) throws DataException { String invitee = groupInviteTransactionData.getInvitee(); - // If member exists then they were added when invite matched join request - if (this.memberExists(invitee)) { + // If member exists and the join request is present then they were added when invite matched join request + if (this.memberExists(invitee) && groupInviteTransactionData.getJoinReference() != null) { // Rebuild join request using cached reference to transaction that created join request. this.rebuildJoinRequest(invitee, groupInviteTransactionData.getJoinReference()); diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index fe001137..2b653ab5 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -76,9 +76,9 @@ public interface ATRepository { * Although expectedValue, if provided, is natively an unsigned long, * the data segment comparison is done via unsigned hex string. */ - public List getMatchingFinalATStates(byte[] codeHash, String buyerAddress, String sellerAddress, Boolean isFinished, - Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight, - Integer limit, Integer offset, Boolean reverse) throws DataException; + public List getMatchingFinalATStates(byte[] codeHash, byte[] buyerPublicKey, byte[] sellerPublicKey, Boolean isFinished, + Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight, + Integer limit, Integer offset, Boolean reverse) throws DataException; /** * Returns final ATStateData for ATs matching codeHash (required) diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 1c0e84e2..4770d29b 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -27,6 +27,10 @@ public interface ArbitraryRepository { public List getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException; + List getLatestArbitraryTransactions() throws DataException; + + List getLatestArbitraryTransactionsByName(String name) throws DataException; + public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException; public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException; @@ -42,7 +46,7 @@ public interface ArbitraryRepository { public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; - public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, List namesFilter, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException; + public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, List keywords, boolean prefixOnly, List namesFilter, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException; List searchArbitraryResourcesSimple( Service service, diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index 71a95428..6310ec02 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -5,6 +5,7 @@ import com.google.common.primitives.Longs; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; +import org.qortal.crypto.Crypto; import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.repository.ATRepository; @@ -403,9 +404,9 @@ public class HSQLDBATRepository implements ATRepository { } @Override - public List getMatchingFinalATStates(byte[] codeHash, String buyerAddress, String sellerAddress, Boolean isFinished, - Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight, - Integer limit, Integer offset, Boolean reverse) throws DataException { + public List getMatchingFinalATStates(byte[] codeHash, byte[] buyerPublicKey, byte[] sellerPublicKey, Boolean isFinished, + Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight, + Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(1024); List bindParams = new ArrayList<>(); @@ -426,9 +427,9 @@ public class HSQLDBATRepository implements ATRepository { // Both must be the same direction (DESC) also sql.append("ORDER BY ATStates.height DESC LIMIT 1) AS FinalATStates "); - // Optional LEFT JOIN with ATTRANSACTIONS for buyerAddress - if (buyerAddress != null && !buyerAddress.isEmpty()) { - sql.append("LEFT JOIN ATTRANSACTIONS tx ON tx.at_address = ATs.AT_address "); + // Optional JOIN with ATTRANSACTIONS for buyerAddress + if (buyerPublicKey != null && buyerPublicKey.length > 0) { + sql.append("JOIN ATTRANSACTIONS tx ON tx.at_address = ATs.AT_address "); } sql.append("WHERE ATs.code_hash = ? "); @@ -450,18 +451,18 @@ public class HSQLDBATRepository implements ATRepository { bindParams.add(rawExpectedValue); } - if (buyerAddress != null && !buyerAddress.isEmpty()) { - sql.append("AND tx.recipient = ? "); - bindParams.add(buyerAddress); + if (buyerPublicKey != null && buyerPublicKey.length > 0 ) { + // the buyer must be the recipient of the transaction and not the creator of the AT + sql.append("AND tx.recipient = ? AND ATs.creator != ? "); + + bindParams.add(Crypto.toAddress(buyerPublicKey)); + bindParams.add(buyerPublicKey); } - if (sellerAddress != null && !sellerAddress.isEmpty()) { - // Convert sellerAddress to publicKey (method depends on your implementation) - AccountData accountData = this.repository.getAccountRepository().getAccount(sellerAddress); - byte[] publicKey = accountData.getPublicKey(); + if (sellerPublicKey != null && sellerPublicKey.length > 0) { sql.append("AND ATs.creator = ? "); - bindParams.add(publicKey); + bindParams.add(sellerPublicKey); } sql.append(" ORDER BY FinalATStates.height "); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 049e98aa..0e15be77 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -7,7 +7,6 @@ import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Service; -import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.data.arbitrary.ArbitraryResourceCache; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceMetadata; @@ -29,6 +28,7 @@ import org.qortal.utils.ListUtils; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -227,6 +227,144 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } } + @Override + public List getLatestArbitraryTransactions() throws DataException { + String sql = "SELECT type, reference, signature, creator, created_when, fee, " + + "tx_group_id, block_height, approval_status, approval_height, " + + "version, nonce, service, size, is_data_raw, data, metadata_hash, " + + "name, identifier, update_method, secret, compression FROM ArbitraryTransactions " + + "JOIN Transactions USING (signature) " + + "WHERE name IS NOT NULL " + + "ORDER BY created_when DESC"; + List arbitraryTransactionData = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return new ArrayList<>(0); + + do { + byte[] reference = resultSet.getBytes(2); + byte[] signature = resultSet.getBytes(3); + byte[] creatorPublicKey = resultSet.getBytes(4); + long timestamp = resultSet.getLong(5); + + Long fee = resultSet.getLong(6); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(7); + + Integer blockHeight = resultSet.getInt(8); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9)); + Integer approvalHeight = resultSet.getInt(10); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + int version = resultSet.getInt(11); + int nonce = resultSet.getInt(12); + int serviceInt = resultSet.getInt(13); + int size = resultSet.getInt(14); + boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false + DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; + byte[] data = resultSet.getBytes(16); + byte[] metadataHash = resultSet.getBytes(17); + String nameResult = resultSet.getString(18); + String identifierResult = resultSet.getString(19); + Method method = Method.valueOf(resultSet.getInt(20)); + byte[] secret = resultSet.getBytes(21); + Compression compression = Compression.valueOf(resultSet.getInt(22)); + // FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls. + + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + version, serviceInt, nonce, size, nameResult, identifierResult, method, secret, + compression, data, dataType, metadataHash, null); + + arbitraryTransactionData.add(transactionData); + } while (resultSet.next()); + + return arbitraryTransactionData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + return new ArrayList<>(0); + } + } + + @Override + public List getLatestArbitraryTransactionsByName( String name ) throws DataException { + String sql = "SELECT type, reference, signature, creator, created_when, fee, " + + "tx_group_id, block_height, approval_status, approval_height, " + + "version, nonce, service, size, is_data_raw, data, metadata_hash, " + + "name, identifier, update_method, secret, compression FROM ArbitraryTransactions " + + "JOIN Transactions USING (signature) " + + "WHERE name = ? " + + "ORDER BY created_when DESC"; + List arbitraryTransactionData = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql, name)) { + if (resultSet == null) + return new ArrayList<>(0); + + do { + byte[] reference = resultSet.getBytes(2); + byte[] signature = resultSet.getBytes(3); + byte[] creatorPublicKey = resultSet.getBytes(4); + long timestamp = resultSet.getLong(5); + + Long fee = resultSet.getLong(6); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(7); + + Integer blockHeight = resultSet.getInt(8); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9)); + Integer approvalHeight = resultSet.getInt(10); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + int version = resultSet.getInt(11); + int nonce = resultSet.getInt(12); + int serviceInt = resultSet.getInt(13); + int size = resultSet.getInt(14); + boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false + DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; + byte[] data = resultSet.getBytes(16); + byte[] metadataHash = resultSet.getBytes(17); + String nameResult = resultSet.getString(18); + String identifierResult = resultSet.getString(19); + Method method = Method.valueOf(resultSet.getInt(20)); + byte[] secret = resultSet.getBytes(21); + Compression compression = Compression.valueOf(resultSet.getInt(22)); + // FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls. + + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + version, serviceInt, nonce, size, nameResult, identifierResult, method, secret, + compression, data, dataType, metadataHash, null); + + arbitraryTransactionData.add(transactionData); + } while (resultSet.next()); + + return arbitraryTransactionData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + return new ArrayList<>(0); + } + } + private ArbitraryTransactionData getSingleTransaction(String name, Service service, Method method, String identifier, boolean firstNotLast) throws DataException { if (name == null || service == null) { // Required fields @@ -724,12 +862,11 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } @Override - public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, + public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, List keywords, boolean prefixOnly, List exactMatchNames, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException { if(Settings.getInstance().isDbCacheEnabled()) { - List list = HSQLDBCacheUtils.callCache( ArbitraryResourceCache.getInstance(), @@ -751,6 +888,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { Optional.ofNullable(description), prefixOnly, Optional.ofNullable(exactMatchNames), + Optional.ofNullable(keywords), defaultResource, Optional.ofNullable(minLevel), Optional.ofNullable(() -> ListUtils.followedNames()), @@ -771,6 +909,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } } + StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); @@ -857,6 +996,26 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { bindParams.add(queryWildcard); } + if (keywords != null && !keywords.isEmpty()) { + List searchKeywords = new ArrayList<>(keywords); + + List conditions = new ArrayList<>(); + List bindValues = new ArrayList<>(); + + for (int i = 0; i < searchKeywords.size(); i++) { + conditions.add("LOWER(description) LIKE ?"); + bindValues.add("%" + searchKeywords.get(i).trim().toLowerCase() + "%"); + } + + String finalCondition = String.join(" OR ", conditions); + sql.append(" AND (").append(finalCondition).append(")"); + + bindParams.addAll(bindValues); + } + + + + // Handle name searches if (names != null && !names.isEmpty()) { sql.append(" AND ("); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java index 24e2da56..46cd7cab 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java @@ -167,6 +167,7 @@ public class HSQLDBCacheUtils { Optional description, boolean prefixOnly, Optional> exactMatchNames, + Optional> keywords, boolean defaultResource, Optional minLevel, Optional>> includeOnly, @@ -182,6 +183,14 @@ public class HSQLDBCacheUtils { // retain only candidates with names Stream stream = candidates.stream().filter(candidate -> candidate.name != null ); + if(after.isPresent()) { + stream = stream.filter( candidate -> candidate.created > after.get().longValue() ); + } + + if(before.isPresent()) { + stream = stream.filter( candidate -> candidate.created < before.get().longValue() ); + } + if(exclude.isPresent()) stream = stream.filter( candidate -> !exclude.get().get().contains( candidate.name )); @@ -207,6 +216,36 @@ public class HSQLDBCacheUtils { stream = filterTerm(title, data -> data.metadata != null ? data.metadata.getTitle() : null, prefixOnly, stream); stream = filterTerm(description, data -> data.metadata != null ? data.metadata.getDescription() : null, prefixOnly, stream); + // New: Filter by keywords if provided + if (keywords.isPresent() && !keywords.get().isEmpty()) { + List searchKeywords = keywords.get().stream() + .map(String::toLowerCase) + .collect(Collectors.toList()); + + stream = stream.filter(candidate -> { + + if (candidate.metadata != null && candidate.metadata.getDescription() != null) { + String descriptionLower = candidate.metadata.getDescription().toLowerCase(); + return searchKeywords.stream().anyMatch(descriptionLower::contains); + } + return false; + }); + } + + if (keywords.isPresent() && !keywords.get().isEmpty()) { + List searchKeywords = keywords.get().stream() + .map(String::toLowerCase) + .collect(Collectors.toList()); + + stream = stream.filter(candidate -> { + if (candidate.metadata != null && candidate.metadata.getDescription() != null) { + String descriptionLower = candidate.metadata.getDescription().toLowerCase(); + return searchKeywords.stream().anyMatch(descriptionLower::contains); + } + return false; + }); + } + // if exact names is set, retain resources with exact names if( exactMatchNames.isPresent() && !exactMatchNames.get().isEmpty()) { @@ -262,15 +301,58 @@ public class HSQLDBCacheUtils { // truncate to limit if( limit.isPresent() && limit.get() > 0 ) stream = stream.limit(limit.get()); - // include metadata - if( includeMetadata.isEmpty() || !includeMetadata.get() ) - stream = stream.peek( candidate -> candidate.metadata = null ); + List listCopy1 = stream.collect(Collectors.toList()); - // include status - if( includeStatus.isEmpty() || !includeStatus.get() ) - stream = stream.peek( candidate -> candidate.status = null); + List listCopy2 = new ArrayList<>(listCopy1.size()); - return stream.collect(Collectors.toList()); + // remove metadata from the first copy + if( includeMetadata.isEmpty() || !includeMetadata.get() ) { + for( ArbitraryResourceData data : listCopy1 ) { + ArbitraryResourceData copy = new ArbitraryResourceData(); + copy.name = data.name; + copy.service = data.service; + copy.identifier = data.identifier; + copy.status = data.status; + copy.metadata = null; + + copy.size = data.size; + copy.created = data.created; + copy.updated = data.updated; + + listCopy2.add(copy); + } + } + // put the list copy 1 into the second copy + else { + listCopy2.addAll(listCopy1); + } + + // remove status from final copy + if( includeStatus.isEmpty() || !includeStatus.get() ) { + + List finalCopy = new ArrayList<>(listCopy2.size()); + + for( ArbitraryResourceData data : listCopy2 ) { + ArbitraryResourceData copy = new ArbitraryResourceData(); + copy.name = data.name; + copy.service = data.service; + copy.identifier = data.identifier; + copy.status = null; + copy.metadata = data.metadata; + + copy.size = data.size; + copy.created = data.created; + copy.updated = data.updated; + + finalCopy.add(copy); + } + + return finalCopy; + } + // keep status included by returning the second copy + else { + return listCopy2; + } } /** diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index f6a4ed5a..21353fb0 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -386,7 +386,7 @@ public class Settings { /** * DB Cache Enabled? */ - private boolean dbCacheEnabled = false; + private boolean dbCacheEnabled = true; /** * DB Cache Thread Priority @@ -508,7 +508,44 @@ public class Settings { */ private boolean connectionPoolMonitorEnabled = false; - // Domain mapping + /** + * Buiild Arbitrary Resources Batch Size + * + * The number resources to batch per iteration when rebuilding. + */ + private int buildArbitraryResourcesBatchSize = 200; + + /** + * Arbitrary Indexing Priority + * + * The thread priority when indexing arbirary resources. + */ + private int arbitraryIndexingPriority = 5; + + /** + * Arbitrary Indexing Frequency (In Minutes) + * + * The frequency at which the arbitrary indices are cached. + */ + private int arbitraryIndexingFrequency = 10; + + private boolean rebuildArbitraryResourceCacheTaskEnabled = false; + + /** + * Rebuild Arbitrary Resource Cache Task Delay (In Minutes) + * + * Waiting period before the first rebuild task is started. + */ + private int rebuildArbitraryResourceCacheTaskDelay = 300; + + /** + * Rebuild Arbitrary Resource Cache Task Period (In Hours) + * + * The frequency the arbitrary resource cache is rebuilt. + */ + private int rebuildArbitraryResourceCacheTaskPeriod = 24; + + // Domain mapping public static class ThreadLimit { private String messageType; private Integer limit; @@ -1353,5 +1390,29 @@ public class Settings { public int getReticulumMaxNetworkThreadPoolSize() { return this.reticulumMaxNetworkThreadPoolSize; + } + + public int getBuildArbitraryResourcesBatchSize() { + return buildArbitraryResourcesBatchSize; + } + + public int getArbitraryIndexingPriority() { + return arbitraryIndexingPriority; + } + + public int getArbitraryIndexingFrequency() { + return arbitraryIndexingFrequency; + } + + public boolean isRebuildArbitraryResourceCacheTaskEnabled() { + return rebuildArbitraryResourceCacheTaskEnabled; + } + + public int getRebuildArbitraryResourceCacheTaskDelay() { + return rebuildArbitraryResourceCacheTaskDelay; + } + + public int getRebuildArbitraryResourceCacheTaskPeriod() { + return rebuildArbitraryResourceCacheTaskPeriod; } } diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index ee9b0b8e..a51913e0 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -9,6 +9,7 @@ import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Service; import org.qortal.block.BlockChain; import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.controller.arbitrary.ArbitraryTransactionDataHashWrapper; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.crypto.Crypto; import org.qortal.crypto.MemoryPoW; @@ -31,8 +32,12 @@ import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.NTP; import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; public class ArbitraryTransaction extends Transaction { @@ -303,8 +308,13 @@ public class ArbitraryTransaction extends Transaction { // Add/update arbitrary resource caches, but don't update the status as this involves time-consuming // disk reads, and is more prone to failure. The status will be updated on metadata retrieval, or when // accessing the resource. - this.updateArbitraryResourceCache(repository); - this.updateArbitraryMetadataCache(repository); + // Also, must add this transaction as a latest transaction, since the it has not been saved to the + // repository yet. + this.updateArbitraryResourceCacheIncludingMetadata( + repository, + Set.of(new ArbitraryTransactionDataHashWrapper(arbitraryTransactionData)), + new HashMap<>(0) + ); repository.saveChanges(); @@ -360,7 +370,10 @@ public class ArbitraryTransaction extends Transaction { * * @throws DataException */ - public void updateArbitraryResourceCache(Repository repository) throws DataException { + public void updateArbitraryResourceCacheIncludingMetadata( + Repository repository, + Set latestTransactionWrappers, + Map resourceByWrapper) throws DataException { // Don't cache resources without a name (such as auto updates) if (arbitraryTransactionData.getName() == null) { return; @@ -385,29 +398,42 @@ public class ArbitraryTransaction extends Transaction { arbitraryResourceData.name = name; arbitraryResourceData.identifier = identifier; - // Get the latest transaction - ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); - if (latestTransactionData == null) { - LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData); - // We don't have a latest transaction, so delete from cache - repository.getArbitraryRepository().delete(arbitraryResourceData); - return; + final ArbitraryTransactionDataHashWrapper wrapper = new ArbitraryTransactionDataHashWrapper(arbitraryTransactionData); + + ArbitraryTransactionData latestTransactionData; + if( latestTransactionWrappers.contains(wrapper)) { + latestTransactionData + = latestTransactionWrappers.stream() + .filter( latestWrapper -> latestWrapper.equals(wrapper)) + .findAny().get() + .getData(); } + else { + // Get the latest transaction + latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); + if (latestTransactionData == null) { + LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData); + // We don't have a latest transaction, so delete from cache + repository.getArbitraryRepository().delete(arbitraryResourceData); + return; + } + } + ArbitraryResourceData existingArbitraryResourceData = resourceByWrapper.get(wrapper); - // Get existing cached entry if it exists - ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository() - .getArbitraryResource(service, name, identifier); - - LOGGER.info("updating existing arbitraryResourceData" + existingArbitraryResourceData); + if( existingArbitraryResourceData == null ) { + // Get existing cached entry if it exists + existingArbitraryResourceData = repository.getArbitraryRepository() + .getArbitraryResource(service, name, identifier); + } // Check for existing cached data if (existingArbitraryResourceData == null) { // Nothing exists yet, so set creation date from the current transaction (it will be reduced later if needed) arbitraryResourceData.created = arbitraryTransactionData.getTimestamp(); arbitraryResourceData.updated = null; - LOGGER.info("updated = null, reason = existingArbitraryResourceData == null" ); } else { + resourceByWrapper.put(wrapper, existingArbitraryResourceData); // An entry already exists - update created time from current transaction if this is older arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp()); @@ -415,22 +441,44 @@ public class ArbitraryTransaction extends Transaction { if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) { // Latest transaction matches created time, so it hasn't been updated arbitraryResourceData.updated = null; - LOGGER.info( - "updated = null, reason: existingArbitraryResourceData.created == latestTransactionData.getTimestamp() == " + - existingArbitraryResourceData.created ); } else { arbitraryResourceData.updated = latestTransactionData.getTimestamp(); - LOGGER.info("setting updated to a non-null value"); } } arbitraryResourceData.size = latestTransactionData.getSize(); - LOGGER.info("saving updated arbitraryResourceData: updated = " + arbitraryResourceData.updated); - // Save repository.getArbitraryRepository().save(arbitraryResourceData); + + // Update metadata for latest transaction if it is local + if (latestTransactionData.getMetadataHash() != null) { + ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature()); + if (metadataFile.exists()) { + ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath()); + try { + transactionMetadata.read(); + + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setArbitraryResourceData(arbitraryResourceData); + metadata.setTitle(transactionMetadata.getTitle()); + metadata.setDescription(transactionMetadata.getDescription()); + metadata.setCategory(transactionMetadata.getCategory()); + metadata.setTags(transactionMetadata.getTags()); + repository.getArbitraryRepository().save(metadata); + + } catch (IOException e) { + // Ignore, as we can add it again later + } + } else { + // We don't have a local copy of this metadata file, so delete it from the cache + // It will be re-added if the file later arrives via the network + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setArbitraryResourceData(arbitraryResourceData); + repository.getArbitraryRepository().delete(metadata); + } + } } public void updateArbitraryResourceStatus(Repository repository) throws DataException { @@ -465,60 +513,4 @@ public class ArbitraryTransaction extends Transaction { repository.getArbitraryRepository().setStatus(arbitraryResourceData, status); } - public void updateArbitraryMetadataCache(Repository repository) throws DataException { - // Get the latest transaction - ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); - if (latestTransactionData == null) { - // We don't have a latest transaction, so give up - return; - } - - Service service = latestTransactionData.getService(); - String name = latestTransactionData.getName(); - String identifier = latestTransactionData.getIdentifier(); - - if (service == null) { - // Unsupported service - ignore this resource - return; - } - - // In the cache we store null identifiers as "default", as it is part of the primary key - if (identifier == null) { - identifier = "default"; - } - - ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); - arbitraryResourceData.service = service; - arbitraryResourceData.name = name; - arbitraryResourceData.identifier = identifier; - - // Update metadata for latest transaction if it is local - if (latestTransactionData.getMetadataHash() != null) { - ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature()); - if (metadataFile.exists()) { - ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath()); - try { - transactionMetadata.read(); - - ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); - metadata.setArbitraryResourceData(arbitraryResourceData); - metadata.setTitle(transactionMetadata.getTitle()); - metadata.setDescription(transactionMetadata.getDescription()); - metadata.setCategory(transactionMetadata.getCategory()); - metadata.setTags(transactionMetadata.getTags()); - repository.getArbitraryRepository().save(metadata); - - } catch (IOException e) { - // Ignore, as we can add it again later - } - } else { - // We don't have a local copy of this metadata file, so delete it from the cache - // It will be re-added if the file later arrives via the network - ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); - metadata.setArbitraryResourceData(arbitraryResourceData); - repository.getArbitraryRepository().delete(metadata); - } - } - } - } diff --git a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java new file mode 100644 index 00000000..17c966fe --- /dev/null +++ b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java @@ -0,0 +1,250 @@ +package org.qortal.utils; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.InvalidFormatException; +import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.api.SearchMode; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataReader; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryDataIndex; +import org.qortal.data.arbitrary.ArbitraryDataIndexDetail; +import org.qortal.data.arbitrary.ArbitraryResourceData; +import org.qortal.data.arbitrary.IndexCache; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class ArbitraryIndexUtils { + + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final Logger LOGGER = LogManager.getLogger(ArbitraryIndexUtils.class); + + public static final String INDEX_CACHE_TIMER = "Arbitrary Index Cache Timer"; + public static final String INDEX_CACHE_TIMER_TASK = "Arbitrary Index Cache Timer Task"; + + public static void startCaching(int priorityRequested, int frequency) { + + Timer timer = buildTimer(INDEX_CACHE_TIMER, priorityRequested); + + TimerTask task = new TimerTask() { + @Override + public void run() { + + Thread.currentThread().setName(INDEX_CACHE_TIMER_TASK); + + try { + fillCache(IndexCache.getInstance()); + } catch (IOException | DataException e) { + LOGGER.error(e.getMessage(), e); + } + } + }; + + // delay 1 second + timer.scheduleAtFixedRate(task, 1_000, frequency * 60_000); + } + + private static void fillCache(IndexCache instance) throws DataException, IOException { + + try (final Repository repository = RepositoryManager.getRepository()) { + + List indexResources + = repository.getArbitraryRepository().searchArbitraryResources( + Service.JSON, + null, + "idx-", + null, + null, + null, + null, + true, + null, + false, + SearchMode.ALL, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + true); + + List indexDetails = new ArrayList<>(); + + LOGGER.debug("processing index resource data: count = " + indexResources.size()); + + // process all index resources + for( ArbitraryResourceData indexResource : indexResources ) { + + try { + LOGGER.debug("processing index resource: name = " + indexResource.name + ", identifier = " + indexResource.identifier); + String json = ArbitraryIndexUtils.getJson(indexResource.name, indexResource.identifier); + + // map the JSON string to a list of Java objects + List indices = OBJECT_MAPPER.readValue(json, new TypeReference>() {}); + + LOGGER.debug("processed indices = " + indices); + + // rank and create index detail for each index in this index resource + for( int rank = 1; rank <= indices.size(); rank++ ) { + + indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1), indexResource.identifier )); + } + } catch (InvalidFormatException e) { + LOGGER.debug("invalid format, skipping: " + indexResource); + } catch (UnrecognizedPropertyException e) { + LOGGER.debug("unrecognized property, skipping " + indexResource); + } + } + + LOGGER.debug("processing indices by term ..."); + Map> indicesByTerm + = indexDetails.stream().collect( + Collectors.toMap( + detail -> detail.term, // map by term + detail -> List.of(detail), // create list for term + (list1, list2) // merge lists for same term + -> Stream.of(list1, list2) + .flatMap(List::stream) + .collect(Collectors.toList()) + ) + ); + + LOGGER.info("processed indices by term: count = " + indicesByTerm.size()); + + // lock, clear old, load new + synchronized( IndexCache.getInstance().getIndicesByTerm() ) { + IndexCache.getInstance().getIndicesByTerm().clear(); + IndexCache.getInstance().getIndicesByTerm().putAll(indicesByTerm); + } + + LOGGER.info("loaded indices by term"); + + LOGGER.debug("processing indices by issuer ..."); + Map> indicesByIssuer + = indexDetails.stream().collect( + Collectors.toMap( + detail -> detail.issuer, // map by issuer + detail -> List.of(detail), // create list for issuer + (list1, list2) // merge lists for same issuer + -> Stream.of(list1, list2) + .flatMap(List::stream) + .collect(Collectors.toList()) + ) + ); + + LOGGER.info("processed indices by issuer: count = " + indicesByIssuer.size()); + + // lock, clear old, load new + synchronized( IndexCache.getInstance().getIndicesByIssuer() ) { + IndexCache.getInstance().getIndicesByIssuer().clear(); + IndexCache.getInstance().getIndicesByIssuer().putAll(indicesByIssuer); + } + + LOGGER.info("loaded indices by issuer"); + } + } + + private static Timer buildTimer( final String name, int priorityRequested) { + // ensure priority is in between 1-10 + final int priority = Math.max(0, Math.min(10, priorityRequested)); + + // Create a custom Timer with updated priority threads + Timer timer = new Timer(true) { // 'true' to make the Timer daemon + @Override + public void schedule(TimerTask task, long delay) { + Thread thread = new Thread(task, name) { + @Override + public void run() { + this.setPriority(priority); + super.run(); + } + }; + thread.setPriority(priority); + thread.start(); + } + }; + return timer; + } + + + public static String getJsonWithExceptionHandling( String name, String identifier ) { + try { + return getJson(name, identifier); + } + catch( Exception e ) { + LOGGER.error(e.getMessage(), e); + return e.getMessage(); + } + } + + public static String getJson(String name, String identifier) throws IOException { + + try { + ArbitraryDataReader arbitraryDataReader + = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, Service.JSON, identifier); + + int attempts = 0; + Integer maxAttempts = 5; + + while (!Controller.isStopping()) { + attempts++; + if (!arbitraryDataReader.isBuilding()) { + try { + arbitraryDataReader.loadSynchronously(false); + break; + } catch (MissingDataException e) { + if (attempts > maxAttempts) { + // Give up after 5 attempts + throw new IOException("Data unavailable. Please try again later."); + } + } + } + Thread.sleep(3000L); + } + + java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); + if (outputPath == null) { + // Assume the resource doesn't exist + throw new IOException( "File not found"); + } + + // No file path supplied - so check if this is a single file resource + String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); + String filepath = files[0]; + + java.nio.file.Path path = Paths.get(outputPath.toString(), filepath); + if (!Files.exists(path)) { + String message = String.format("No file exists at filepath: %s", filepath); + throw new IOException( message ); + } + + String data = Files.readString(path); + + return data; + } catch (Exception e) { + throw new IOException(String.format("Unable to load %s %s: %s", Service.JSON, name, e.getMessage())); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index f641255f..c860a034 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -24,6 +24,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; @@ -72,23 +73,23 @@ public class ArbitraryTransactionUtils { return latestPut; } - public static boolean hasMoreRecentPutTransaction(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { + public static Optional hasMoreRecentPutTransaction(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { byte[] signature = arbitraryTransactionData.getSignature(); if (signature == null) { // We can't make a sensible decision without a signature // so it's best to assume there is nothing newer - return false; + return Optional.empty(); } ArbitraryTransactionData latestPut = ArbitraryTransactionUtils.fetchLatestPut(repository, arbitraryTransactionData); if (latestPut == null) { - return false; + return Optional.empty(); } // If the latest PUT transaction has a newer timestamp, it will override the existing transaction // Any data relating to the older transaction is no longer needed boolean hasNewerPut = (latestPut.getTimestamp() > arbitraryTransactionData.getTimestamp()); - return hasNewerPut; + return hasNewerPut ? Optional.of(latestPut) : Optional.empty(); } public static boolean completeFileExists(ArbitraryTransactionData transactionData) throws DataException { @@ -208,7 +209,15 @@ public class ArbitraryTransactionUtils { return ArbitraryTransactionUtils.isFileRecent(filePath, now, cleanupAfter); } - public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException { + /** + * + * @param arbitraryTransactionData + * @param now + * @param cleanupAfter + * @return true if file is deleted, otherwise return false + * @throws DataException + */ + public static boolean deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException { byte[] completeHash = arbitraryTransactionData.getData(); byte[] signature = arbitraryTransactionData.getSignature(); @@ -219,6 +228,11 @@ public class ArbitraryTransactionUtils { "if needed", Base58.encode(completeHash)); arbitraryDataFile.delete(); + + return true; + } + else { + return false; } } diff --git a/src/main/java/org/qortal/utils/Blocks.java b/src/main/java/org/qortal/utils/Blocks.java index 54ad86da..0681af75 100644 --- a/src/main/java/org/qortal/utils/Blocks.java +++ b/src/main/java/org/qortal/utils/Blocks.java @@ -53,10 +53,10 @@ public class Blocks { // all minting group member addresses List mintingGroupAddresses - = repository.getGroupRepository() - .getGroupMembers(BlockChain.getInstance().getMintingGroupId()).stream() - .map(GroupMemberData::getMember) - .collect(Collectors.toList()); + = Groups.getAllMembers( + repository.getGroupRepository(), + Groups.getGroupIdsToMint(BlockChain.getInstance(), blockData.getHeight()) + ); // all names, indexed by address Map nameByAddress diff --git a/src/main/java/org/qortal/utils/Groups.java b/src/main/java/org/qortal/utils/Groups.java new file mode 100644 index 00000000..131bc93e --- /dev/null +++ b/src/main/java/org/qortal/utils/Groups.java @@ -0,0 +1,122 @@ +package org.qortal.utils; + +import org.qortal.block.BlockChain; +import org.qortal.data.group.GroupAdminData; +import org.qortal.data.group.GroupMemberData; +import org.qortal.repository.DataException; +import org.qortal.repository.GroupRepository; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Class Groups + * + * A utility class for group related functionality. + */ +public class Groups { + + /** + * Does the member exist in any of these groups? + * + * @param groupRepository the group data repository + * @param groupsIds the group Ids to look for the address + * @param address the address + * + * @return true if the address is in any of the groups listed otherwise false + * @throws DataException + */ + public static boolean memberExistsInAnyGroup(GroupRepository groupRepository, List groupsIds, String address) throws DataException { + + // if any of the listed groups have the address as a member, then return true + for( Integer groupIdToMint : groupsIds) { + if( groupRepository.memberExists(groupIdToMint, address) ) { + return true; + } + } + + // if none of the listed groups have the address as a member, then return false + return false; + } + + /** + * Get All Members + * + * Get all the group members from a list of groups. + * + * @param groupRepository the group data repository + * @param groupIds the list of group Ids to look at + * + * @return the list of all members belonging to any of the groups, no duplicates + * @throws DataException + */ + public static List getAllMembers( GroupRepository groupRepository, List groupIds ) throws DataException { + // collect all the members in a set, the set keeps out duplicates + Set allMembers = new HashSet<>(); + + // add all members from each group to the all members set + for( int groupId : groupIds ) { + allMembers.addAll( groupRepository.getGroupMembers(groupId).stream().map(GroupMemberData::getMember).collect(Collectors.toList())); + } + + return new ArrayList<>(allMembers); + } + + /** + * Get All Admins + * + * Get all the admins from a list of groups. + * + * @param groupRepository the group data repository + * @param groupIds the list of group Ids to look at + * + * @return the list of all admins to any of the groups, no duplicates + * @throws DataException + */ + public static List getAllAdmins( GroupRepository groupRepository, List groupIds ) throws DataException { + // collect all the admins in a set, the set keeps out duplicates + Set allAdmins = new HashSet<>(); + + // collect admins for each group + for( int groupId : groupIds ) { + allAdmins.addAll( groupRepository.getGroupAdmins(groupId).stream().map(GroupAdminData::getAdmin).collect(Collectors.toList()) ); + } + + return new ArrayList<>(allAdmins); + } + + /** + * Get Group Ids To Mint + * + * @param blockchain the blockchain + * @param blockchainHeight the block height to mint + * + * @return the group Ids for the minting groups at the height given + */ + public static List getGroupIdsToMint(BlockChain blockchain, int blockchainHeight) { + + // sort heights lowest to highest + Comparator compareByHeight = Comparator.comparingInt(entry -> entry.height); + + // sort heights highest to lowest + Comparator compareByHeightReversed = compareByHeight.reversed(); + + // get highest height that is less than the blockchain height + Optional ids = blockchain.getMintingGroupIds().stream() + .filter(entry -> entry.height < blockchainHeight) + .sorted(compareByHeightReversed) + .findFirst(); + + if( ids.isPresent()) { + return ids.get().ids; + } + else { + return new ArrayList<>(0); + } + } +} \ No newline at end of file diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 40ab983a..3264b670 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -38,7 +38,9 @@ "blockRewardBatchStartHeight": 1508000, "blockRewardBatchSize": 1000, "blockRewardBatchAccountsBlockCount": 25, - "mintingGroupId": 694, + "mintingGroupIds": [ + { "height": 0, "ids": [ 694 ]} + ], "rewardsByHeight": [ { "height": 1, "reward": 5.00 }, { "height": 259201, "reward": 4.75 }, diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js index 25656370..d7222750 100644 --- a/src/main/resources/q-apps/q-apps.js +++ b/src/main/resources/q-apps/q-apps.js @@ -84,6 +84,7 @@ isDOMContentLoaded: isDOMContentLoaded ? true : false function handleQDNResourceDisplayed(pathurl, isDOMContentLoaded) { // make sure that an empty string the root path +if(pathurl?.startsWith('/render/hash/')) return; const path = pathurl || '/' if (!isManualNavigation) { isManualNavigation = true @@ -284,11 +285,9 @@ window.addEventListener("message", async (event) => { return; } - console.log("Core received action: " + JSON.stringify(event.data.action)); - let url; let data = event.data; - + let identifier; switch (data.action) { case "GET_ACCOUNT_DATA": return httpGetAsyncWithEvent(event, "/addresses/" + data.address); @@ -383,6 +382,7 @@ window.addEventListener("message", async (event) => { if (data.identifier != null) url = url.concat("&identifier=" + data.identifier); if (data.name != null) url = url.concat("&name=" + data.name); if (data.names != null) data.names.forEach((x, i) => url = url.concat("&name=" + x)); + if (data.keywords != null) data.keywords.forEach((x, i) => url = url.concat("&keywords=" + x)); if (data.title != null) url = url.concat("&title=" + data.title); if (data.description != null) url = url.concat("&description=" + data.description); if (data.prefix != null) url = url.concat("&prefix=" + new Boolean(data.prefix).toString()); @@ -419,7 +419,7 @@ window.addEventListener("message", async (event) => { return httpGetAsyncWithEvent(event, url); case "GET_QDN_RESOURCE_PROPERTIES": - let identifier = (data.identifier != null) ? data.identifier : "default"; + identifier = (data.identifier != null) ? data.identifier : "default"; url = "/arbitrary/resource/properties/" + data.service + "/" + data.name + "/" + identifier; return httpGetAsyncWithEvent(event, url); @@ -456,7 +456,7 @@ window.addEventListener("message", async (event) => { return httpGetAsyncWithEvent(event, url); case "GET_AT": - url = "/at" + data.atAddress; + url = "/at/" + data.atAddress; return httpGetAsyncWithEvent(event, url); case "GET_AT_DATA": @@ -473,7 +473,7 @@ window.addEventListener("message", async (event) => { case "FETCH_BLOCK": if (data.signature != null) { - url = "/blocks/" + data.signature; + url = "/blocks/signature/" + data.signature; } else if (data.height != null) { url = "/blocks/byheight/" + data.height; } @@ -694,6 +694,7 @@ const qortalRequestWithTimeout = (request, timeout) => * Send current page details to UI */ document.addEventListener('DOMContentLoaded', (event) => { + resetVariables() qortalRequest({ action: "QDN_RESOURCE_DISPLAYED", @@ -712,6 +713,7 @@ resetVariables() * Handle app navigation */ navigation.addEventListener('navigate', (event) => { + const url = new URL(event.destination.url); let fullpath = url.pathname + url.hash; diff --git a/src/test/java/org/qortal/test/api/CrossChainUtilsTests.java b/src/test/java/org/qortal/test/api/CrossChainUtilsTests.java index 0e4a6f07..5c67267f 100644 --- a/src/test/java/org/qortal/test/api/CrossChainUtilsTests.java +++ b/src/test/java/org/qortal/test/api/CrossChainUtilsTests.java @@ -3,10 +3,15 @@ package org.qortal.test.api; import org.json.simple.JSONObject; import org.junit.Assert; import org.junit.Test; +import org.qortal.api.model.CrossChainTradeLedgerEntry; import org.qortal.api.resource.CrossChainUtils; import org.qortal.test.common.ApiCommon; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; public class CrossChainUtilsTests extends ApiCommon { @@ -137,4 +142,53 @@ public class CrossChainUtilsTests extends ApiCommon { Assert.assertEquals(5, versionDecimal, 0.001); Assert.assertFalse(thrown); } + + @Test + public void testWriteToLedgerHeaderOnly() throws IOException { + CrossChainUtils.writeToLedger(new PrintWriter(System.out), new ArrayList<>()); + } + + @Test + public void testWriteToLedgerOneRow() throws IOException { + CrossChainUtils.writeToLedger( + new PrintWriter(System.out), + List.of( + new CrossChainTradeLedgerEntry( + "QORT", + "LTC", + 1000, + 0, + "LTC", + 1, + System.currentTimeMillis()) + ) + ); + } + + @Test + public void testWriteToLedgerTwoRows() throws IOException { + CrossChainUtils.writeToLedger( + new PrintWriter(System.out), + List.of( + new CrossChainTradeLedgerEntry( + "QORT", + "LTC", + 1000, + 0, + "LTC", + 1, + System.currentTimeMillis() + ), + new CrossChainTradeLedgerEntry( + "LTC", + "QORT", + 1, + 0, + "LTC", + 1000, + System.currentTimeMillis() + ) + ) + ); + } } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java index c05ceabf..33375b62 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java @@ -145,56 +145,6 @@ public class ArbitraryDataStorageCapacityTests extends Common { } } - @Test - public void testDeleteRandomFilesForName() throws DataException, IOException, InterruptedException, IllegalAccessException { - try (final Repository repository = RepositoryManager.getRepository()) { - String identifier = null; // Not used for this test - Service service = Service.ARBITRARY_DATA; - int chunkSize = 100; - int dataLength = 900; // Actual data length will be longer due to encryption - - // Set originalCopyIndicatorFileEnabled to false, otherwise nothing will be deleted as it all originates from this node - FieldUtils.writeField(Settings.getInstance(), "originalCopyIndicatorFileEnabled", false, true); - - // Alice hosts some data (with 10 chunks) - PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); - String aliceName = "alice"; - RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), aliceName, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); - TransactionUtils.signAndMint(repository, transactionData, alice); - Path alicePath = ArbitraryUtils.generateRandomDataPath(dataLength); - ArbitraryDataFile aliceArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(alice.getPublicKey()), alicePath, aliceName, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize); - - // Bob hosts some data too (also with 10 chunks) - PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); - String bobName = "bob"; - transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); - TransactionUtils.signAndMint(repository, transactionData, bob); - Path bobPath = ArbitraryUtils.generateRandomDataPath(dataLength); - ArbitraryDataFile bobArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(bob.getPublicKey()), bobPath, bobName, identifier, ArbitraryTransactionData.Method.PUT, service, bob, chunkSize); - - // All 20 chunks should exist - assertEquals(10, aliceArbitraryDataFile.chunkCount()); - assertTrue(aliceArbitraryDataFile.allChunksExist()); - assertEquals(10, bobArbitraryDataFile.chunkCount()); - assertTrue(bobArbitraryDataFile.allChunksExist()); - - // Now pretend that Bob has reached his storage limit - this should delete random files - // Run it 10 times to remove the likelihood of the randomizer always picking Alice's files - for (int i=0; i<10; i++) { - ArbitraryDataCleanupManager.getInstance().storageLimitReachedForName(repository, bobName); - } - - // Alice should still have all chunks - assertTrue(aliceArbitraryDataFile.allChunksExist()); - - // Bob should be missing some chunks - assertFalse(bobArbitraryDataFile.allChunksExist()); - - } - } - private void deleteListsDirectory() { // Delete lists directory if exists Path listsPath = Paths.get(Settings.getInstance().getListsPath()); diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java index 1d8f23b3..1968eb60 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java @@ -73,14 +73,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.FOLLOWED_OR_VIEWED, Settings.getInstance().getStoragePolicy()); assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We should store but not pre-fetch data for this transaction assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); } } @@ -108,14 +108,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy()); assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We shouldn't store or pre-fetch data for this transaction assertFalse(storageManager.canStoreData(arbitraryTransactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); } } @@ -143,14 +143,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store but not pre-fetch data for this transaction assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy()); assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We should store but not pre-fetch data for this transaction assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); } } @@ -178,14 +178,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy()); assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We should store and pre-fetch data for this transaction assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); } } @@ -213,14 +213,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We shouldn't store or pre-fetch data for this transaction assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy()); assertFalse(storageManager.canStoreData(arbitraryTransactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We shouldn't store or pre-fetch data for this transaction assertFalse(storageManager.canStoreData(arbitraryTransactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); } } @@ -236,7 +236,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store but not pre-fetch data for this transaction assertTrue(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, transactionData).isPass()); } } diff --git a/src/test/java/org/qortal/test/repository/HSQLDBCacheUtilsTests.java b/src/test/java/org/qortal/test/repository/HSQLDBCacheUtilsTests.java index 7a75dea7..6a984b1d 100644 --- a/src/test/java/org/qortal/test/repository/HSQLDBCacheUtilsTests.java +++ b/src/test/java/org/qortal/test/repository/HSQLDBCacheUtilsTests.java @@ -26,6 +26,7 @@ public class HSQLDBCacheUtilsTests { private static final String DESCRIPTION = "description"; private static final String PREFIX_ONLY = "prefixOnly"; private static final String EXACT_MATCH_NAMES = "exactMatchNames"; + private static final String KEYWORDS = "keywords"; private static final String DEFAULT_RESOURCE = "defaultResource"; private static final String MODE = "mode"; private static final String MIN_LEVEL = "minLevel"; @@ -299,6 +300,19 @@ public class HSQLDBCacheUtilsTests { ); } + @Test + public void testAfterNegative() { + ArbitraryResourceData data = new ArbitraryResourceData(); + data.created = 10L; + data.name = "Joe"; + + filterListByMap( + List.of(data), + NAME_LEVEL, new HashMap<>(Map.of(AFTER, 11L)), + 0 + ); + } + @Test public void testBeforePositive(){ ArbitraryResourceData data = new ArbitraryResourceData(); @@ -312,6 +326,19 @@ public class HSQLDBCacheUtilsTests { ); } + @Test + public void testBeforeNegative(){ + ArbitraryResourceData data = new ArbitraryResourceData(); + data.created = 10L; + data.name = "Joe"; + + filterListByMap( + List.of(data), + NAME_LEVEL, new HashMap<>(Map.of(BEFORE, 9L)), + 0 + ); + } + @Test public void testTitlePositive() { @@ -342,6 +369,25 @@ public class HSQLDBCacheUtilsTests { ); } + @Test + public void testMetadataNullificationBugSolution(){ + + ArbitraryResourceData data = new ArbitraryResourceData(); + data.metadata = new ArbitraryResourceMetadata(); + data.metadata.setDescription("Once upon a time."); + data.name = "Joe"; + + List list = List.of(data); + + filterListByMap( + List.of(data), + NAME_LEVEL, new HashMap<>(Map.of(DESCRIPTION, "Once upon a time.")), + 1 + ); + + Assert.assertNotNull(data.metadata); + } + @Test public void testMinLevelPositive() { @@ -615,6 +661,7 @@ public class HSQLDBCacheUtilsTests { Optional description = Optional.ofNullable((String) valueByKey.get(DESCRIPTION)); boolean prefixOnly = valueByKey.containsKey(PREFIX_ONLY); Optional> exactMatchNames = Optional.ofNullable((List) valueByKey.get(EXACT_MATCH_NAMES)); + Optional> keywords = Optional.ofNullable((List) valueByKey.get(KEYWORDS)); boolean defaultResource = valueByKey.containsKey(DEFAULT_RESOURCE); Optional mode = Optional.of((SearchMode) valueByKey.getOrDefault(MODE, SearchMode.ALL)); Optional minLevel = Optional.ofNullable((Integer) valueByKey.get(MIN_LEVEL)); @@ -641,6 +688,7 @@ public class HSQLDBCacheUtilsTests { description, prefixOnly, exactMatchNames, + keywords, defaultResource, minLevel, followedOnly, diff --git a/src/test/java/org/qortal/test/utils/GroupsTestUtils.java b/src/test/java/org/qortal/test/utils/GroupsTestUtils.java new file mode 100644 index 00000000..52f106a7 --- /dev/null +++ b/src/test/java/org/qortal/test/utils/GroupsTestUtils.java @@ -0,0 +1,102 @@ +package org.qortal.test.utils; + +import org.qortal.account.PrivateKeyAccount; +import org.qortal.data.transaction.CreateGroupTransactionData; +import org.qortal.data.transaction.GroupInviteTransactionData; +import org.qortal.data.transaction.JoinGroupTransactionData; +import org.qortal.data.transaction.LeaveGroupTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.test.common.TransactionUtils; +import org.qortal.test.common.transaction.TestTransaction; + +/** + * Class GroupsTestUtils + * + * Utility methods for testing the Groups class. + */ +public class GroupsTestUtils { + + /** + * Create Group + * + * @param repository the data repository + * @param owner the group owner + * @param groupName the group name + * @param isOpen true if the group is public, false for private + * + * @return the group Id + * @throws DataException + */ + public static Integer createGroup(Repository repository, PrivateKeyAccount owner, String groupName, boolean isOpen) throws DataException { + String description = groupName + " (description)"; + + Group.ApprovalThreshold approvalThreshold = Group.ApprovalThreshold.ONE; + int minimumBlockDelay = 10; + int maximumBlockDelay = 1440; + + CreateGroupTransactionData transactionData = new CreateGroupTransactionData(TestTransaction.generateBase(owner), groupName, description, isOpen, approvalThreshold, minimumBlockDelay, maximumBlockDelay); + TransactionUtils.signAndMint(repository, transactionData, owner); + + return repository.getGroupRepository().fromGroupName(groupName).getGroupId(); + } + + /** + * Join Group + * + * @param repository the data repository + * @param joiner the address for the account joining the group + * @param groupId the Id for the group to join + * + * @throws DataException + */ + public static void joinGroup(Repository repository, PrivateKeyAccount joiner, int groupId) throws DataException { + JoinGroupTransactionData transactionData = new JoinGroupTransactionData(TestTransaction.generateBase(joiner), groupId); + TransactionUtils.signAndMint(repository, transactionData, joiner); + } + + /** + * Group Invite + * + * @param repository the data repository + * @param admin the admin account to sign the invite + * @param groupId the Id of the group to invite to + * @param invitee the recipient address for the invite + * @param timeToLive the time length of the invite + * + * @throws DataException + */ + public static void groupInvite(Repository repository, PrivateKeyAccount admin, int groupId, String invitee, int timeToLive) throws DataException { + GroupInviteTransactionData transactionData = new GroupInviteTransactionData(TestTransaction.generateBase(admin), groupId, invitee, timeToLive); + TransactionUtils.signAndMint(repository, transactionData, admin); + } + + /** + * Leave Group + * + * @param repository the data repository + * @param leaver the account leaving + * @param groupId the Id of the group being left + * + * @throws DataException + */ + public static void leaveGroup(Repository repository, PrivateKeyAccount leaver, int groupId) throws DataException { + LeaveGroupTransactionData transactionData = new LeaveGroupTransactionData(TestTransaction.generateBase(leaver), groupId); + TransactionUtils.signAndMint(repository, transactionData, leaver); + } + + /** + * Is Member? + * + * @param repository the data repository + * @param address the account address + * @param groupId the group Id + * + * @return true if the account is a member of the group, otherwise false + * @throws DataException + */ + public static boolean isMember(Repository repository, String address, int groupId) throws DataException { + return repository.getGroupRepository().memberExists(groupId, address); + } +} \ No newline at end of file diff --git a/src/test/java/org/qortal/test/utils/GroupsTests.java b/src/test/java/org/qortal/test/utils/GroupsTests.java new file mode 100644 index 00000000..c9a69f1f --- /dev/null +++ b/src/test/java/org/qortal/test/utils/GroupsTests.java @@ -0,0 +1,199 @@ +package org.qortal.test.utils; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.block.Block; +import org.qortal.block.BlockChain; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.utils.Groups; + +import java.util.List; + +import static org.junit.Assert.*; + +public class GroupsTests extends Common { + + public static final String ALICE = "alice"; + public static final String BOB = "bob"; + public static final String CHLOE = "chloe"; + public static final String DILBERT = "dilbert"; + + + private static final int HEIGHT_1 = 5; + private static final int HEIGHT_2 = 8; + private static final int HEIGHT_3 = 12; + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @After + public void afterTest() throws DataException { + Common.orphanCheck(); + } + + @Test + public void testGetGroupIdsToMintSimple() { + List ids = Groups.getGroupIdsToMint(BlockChain.getInstance(), 0); + + Assert.assertNotNull(ids); + Assert.assertEquals(0, ids.size()); + } + + @Test + public void testGetGroupIdsToMintComplex() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + Block block1 = BlockUtils.mintBlocks(repository, HEIGHT_1); + int height1 = block1.getBlockData().getHeight().intValue(); + assertEquals(HEIGHT_1 + 1, height1); + + List ids1 = Groups.getGroupIdsToMint(BlockChain.getInstance(), height1); + + Assert.assertEquals(1, ids1.size() ); + Assert.assertTrue( ids1.contains( 694 ) ); + + Block block2 = BlockUtils.mintBlocks(repository, HEIGHT_2 - HEIGHT_1); + int height2 = block2.getBlockData().getHeight().intValue(); + assertEquals( HEIGHT_2 + 1, height2); + + List ids2 = Groups.getGroupIdsToMint(BlockChain.getInstance(), height2); + + Assert.assertEquals(2, ids2.size() ); + + Assert.assertTrue( ids2.contains( 694 ) ); + Assert.assertTrue( ids2.contains( 800 ) ); + + Block block3 = BlockUtils.mintBlocks(repository, HEIGHT_3 - HEIGHT_2); + int height3 = block3.getBlockData().getHeight().intValue(); + assertEquals( HEIGHT_3 + 1, height3); + + List ids3 = Groups.getGroupIdsToMint(BlockChain.getInstance(), height3); + + Assert.assertEquals( 1, ids3.size() ); + + Assert.assertTrue( ids3.contains( 800 ) ); + } + } + + @Test + public void testMemberExistsInAnyGroupSimple() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); + + // Create group + int groupId = GroupsTestUtils.createGroup(repository, alice, "closed-group", false); + + // Confirm Bob is not a member + Assert.assertFalse( Groups.memberExistsInAnyGroup(repository.getGroupRepository(), List.of(groupId), bob.getAddress()) ); + + // Bob to join + GroupsTestUtils.joinGroup(repository, bob, groupId); + + // Confirm Bob still not a member + assertFalse(GroupsTestUtils.isMember(repository, bob.getAddress(), groupId)); + + // Have Alice 'invite' Bob to confirm membership + GroupsTestUtils.groupInvite(repository, alice, groupId, bob.getAddress(), 0); // non-expiring invite + + // Confirm Bob now a member + Assert.assertTrue( Groups.memberExistsInAnyGroup(repository.getGroupRepository(), List.of(groupId), bob.getAddress()) ); + } + } + + @Test + public void testGroupsListedFunctionality() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + + PrivateKeyAccount alice = Common.getTestAccount(repository, ALICE); + PrivateKeyAccount bob = Common.getTestAccount(repository, BOB); + PrivateKeyAccount chloe = Common.getTestAccount(repository, CHLOE); + PrivateKeyAccount dilbert = Common.getTestAccount(repository, DILBERT); + + // Create groups + int group1Id = GroupsTestUtils.createGroup(repository, alice, "group-1", false); + int group2Id = GroupsTestUtils.createGroup(repository, bob, "group-2", false); + + // test memberExistsInAnyGroup + Assert.assertTrue(Groups.memberExistsInAnyGroup(repository.getGroupRepository(), List.of(group1Id, group2Id), alice.getAddress())); + Assert.assertFalse(Groups.memberExistsInAnyGroup(repository.getGroupRepository(), List.of(group1Id, group2Id), chloe.getAddress())); + + // alice is a member + Assert.assertTrue(GroupsTestUtils.isMember(repository, alice.getAddress(), group1Id)); + List allMembersBeforeJoin = Groups.getAllMembers(repository.getGroupRepository(), List.of(group1Id)); + + // assert one member + Assert.assertNotNull(allMembersBeforeJoin); + Assert.assertEquals(1, allMembersBeforeJoin.size()); + + List allAdminsBeforeJoin = Groups.getAllAdmins(repository.getGroupRepository(), List.of(group1Id)); + + // assert one admin + Assert.assertNotNull(allAdminsBeforeJoin); + Assert.assertEquals( 1, allAdminsBeforeJoin.size()); + + // Bob to join + GroupsTestUtils.joinGroup(repository, bob, group1Id); + + // Have Alice 'invite' Bob to confirm membership + GroupsTestUtils.groupInvite(repository, alice, group1Id, bob.getAddress(), 0); // non-expiring invite + + List allMembersAfterJoin = Groups.getAllMembers(repository.getGroupRepository(), List.of(group1Id)); + + // alice and bob are members + Assert.assertNotNull(allMembersAfterJoin); + Assert.assertEquals(2, allMembersAfterJoin.size()); + + List allAdminsAfterJoin = Groups.getAllAdmins(repository.getGroupRepository(), List.of(group1Id)); + + // assert still one admin + Assert.assertNotNull(allAdminsAfterJoin); + Assert.assertEquals(1, allAdminsAfterJoin.size()); + + List allAdminsFor2Groups = Groups.getAllAdmins(repository.getGroupRepository(), List.of(group1Id, group2Id)); + + // assert 2 admins when including the second group + Assert.assertNotNull(allAdminsFor2Groups); + Assert.assertEquals(2, allAdminsFor2Groups.size()); + + List allMembersFor2Groups = Groups.getAllMembers(repository.getGroupRepository(), List.of(group1Id, group2Id)); + + // assert 2 members when including the seconds group + Assert.assertNotNull(allMembersFor2Groups); + Assert.assertEquals(2, allMembersFor2Groups.size()); + + GroupsTestUtils.leaveGroup(repository, bob, group1Id); + + List allMembersForAfterBobLeavesGroup1InAllGroups = Groups.getAllMembers(repository.getGroupRepository(), List.of(group1Id, group2Id)); + + // alice and bob are members of one group still + Assert.assertNotNull(allMembersForAfterBobLeavesGroup1InAllGroups); + Assert.assertEquals(2, allMembersForAfterBobLeavesGroup1InAllGroups.size()); + + GroupsTestUtils.groupInvite(repository, alice, group1Id, chloe.getAddress(), 3600); + GroupsTestUtils.groupInvite(repository, bob, group2Id, chloe.getAddress(), 3600); + + GroupsTestUtils.joinGroup(repository, chloe, group1Id); + GroupsTestUtils.joinGroup(repository, chloe, group2Id); + + List allMembersAfterDilbert = Groups.getAllMembers((repository.getGroupRepository()), List.of(group1Id, group2Id)); + + // 3 accounts are now members of one group or another + Assert.assertNotNull(allMembersAfterDilbert); + Assert.assertEquals(3, allMembersAfterDilbert.size()); + } + } + +} \ No newline at end of file diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index 4d1d6240..5395116f 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -31,6 +31,12 @@ "blockRewardBatchStartHeight": 999999000, "blockRewardBatchSize": 10, "blockRewardBatchAccountsBlockCount": 3, + "mintingGroupIds": [ + { "height": 0, "ids": []}, + { "height": 5, "ids": [694]}, + { "height": 8, "ids": [694, 800]}, + { "height": 12, "ids": [800]} + ], "rewardsByHeight": [ { "height": 1, "reward": 100 }, { "height": 11, "reward": 10 },