Qortal/qortal
1
0
Fork 0
mirror of https://github.com/Qortal/qortal.git synced 2025-04-14 23:35:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #250 from kennycud/master

Browse Source 
Merging current 'test release' from kennycud repo after extensive testing by community.
This commit is contained in:
crowetic 2025-04-08 08:34:07 -07:00 committed by GitHub
commit 0b37666d2b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
51 changed files with 2678 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/org/qortal/account/Account.java
View File

@ -14,6 +14,7 @@ import org.qortal.repository.NameRepository;
import org.qortal.repository.Repository;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
import org.qortal.utils.Groups;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@ -227,7 +228,7 @@ public class Account {
}
int level = accountData.getLevel();
int groupIdToMint = BlockChain.getInstance().getMintingGroupId();
List<Integer> groupIdsToMint = Groups.getGroupIdsToMint( BlockChain.getInstance(), blockchainHeight );
int nameCheckHeight = BlockChain.getInstance().getOnlyMintWithNameHeight();
int groupCheckHeight = BlockChain.getInstance().getGroupMemberCheckHeight();
int removeNameCheckHeight = BlockChain.getInstance().getRemoveOnlyMintWithNameHeight();
@ -261,9 +262,9 @@ public class Account {
if (blockchainHeight >= groupCheckHeight && blockchainHeight < removeNameCheckHeight) {
List<NameData> myName = nameRepository.getNamesByOwner(myAddress);
if (Account.isFounder(accountData.getFlags())) {
return accountData.getBlocksMintedPenalty() == 0 && !myName.isEmpty() && (isGroupValidated || groupRepository.memberExists(groupIdToMint, myAddress));
return accountData.getBlocksMintedPenalty() == 0 && !myName.isEmpty() && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
} else {
return level >= levelToMint && !myName.isEmpty() && (isGroupValidated || groupRepository.memberExists(groupIdToMint, myAddress));
return level >= levelToMint && !myName.isEmpty() && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
}
}
@ -272,9 +273,9 @@ public class Account {
// Account's address is a member of the minter group
if (blockchainHeight >= removeNameCheckHeight) {
if (Account.isFounder(accountData.getFlags())) {
return accountData.getBlocksMintedPenalty() == 0 && (isGroupValidated || groupRepository.memberExists(groupIdToMint, myAddress));
return accountData.getBlocksMintedPenalty() == 0 && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
} else {
return level >= levelToMint && (isGroupValidated || groupRepository.memberExists(groupIdToMint, myAddress));
return level >= levelToMint && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
}
}

1
src/main/java/org/qortal/api/ApiService.java
View File

@ -194,6 +194,7 @@ public class ApiService {
context.addServlet(AdminStatusWebSocket.class, "/websockets/admin/status");
context.addServlet(BlocksWebSocket.class, "/websockets/blocks");
context.addServlet(DataMonitorSocket.class, "/websockets/datamonitor");
context.addServlet(ActiveChatsWebSocket.class, "/websockets/chat/active/*");
context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages");
context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers");

72
src/main/java/org/qortal/api/model/CrossChainTradeLedgerEntry.java Normal file
View File

@ -0,0 +1,72 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.data.crosschain.CrossChainTradeData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainTradeLedgerEntry {
private String market;
private String currency;
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long quantity;
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long feeAmount;
private String feeCurrency;
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long totalPrice;
private long tradeTimestamp;
protected CrossChainTradeLedgerEntry() {
/* For JAXB */
}
public CrossChainTradeLedgerEntry(String market, String currency, long quantity, long feeAmount, String feeCurrency, long totalPrice, long tradeTimestamp) {
this.market = market;
this.currency = currency;
this.quantity = quantity;
this.feeAmount = feeAmount;
this.feeCurrency = feeCurrency;
this.totalPrice = totalPrice;
this.tradeTimestamp = tradeTimestamp;
}
public String getMarket() {
return market;
}
public String getCurrency() {
return currency;
}
public long getQuantity() {
return quantity;
}
public long getFeeAmount() {
return feeAmount;
}
public String getFeeCurrency() {
return feeCurrency;
}
public long getTotalPrice() {
return totalPrice;
}
public long getTradeTimestamp() {
return tradeTimestamp;
}
}

94
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -33,9 +33,13 @@ import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
import org.qortal.controller.arbitrary.ArbitraryMetadataManager;
import org.qortal.data.account.AccountData;
import org.qortal.data.arbitrary.ArbitraryCategoryInfo;
import org.qortal.data.arbitrary.ArbitraryDataIndexDetail;
import org.qortal.data.arbitrary.ArbitraryDataIndexScoreKey;
import org.qortal.data.arbitrary.ArbitraryDataIndexScorecard;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.data.arbitrary.IndexCache;
import org.qortal.data.naming.NameData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
@ -69,8 +73,11 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
@Path("/arbitrary")
@Tag(name = "Arbitrary")
@ -172,6 +179,7 @@ public class ArbitraryResource {
@Parameter(description = "Name (searches name field only)") @QueryParam("name") List<String> names,
@Parameter(description = "Title (searches title metadata field only)") @QueryParam("title") String title,
@Parameter(description = "Description (searches description metadata field only)") @QueryParam("description") String description,
@Parameter(description = "Keyword (searches description metadata field by keywords)") @QueryParam("keywords") List<String> keywords,
@Parameter(description = "Prefix only (if true, only the beginning of fields are matched)") @QueryParam("prefix") Boolean prefixOnly,
@Parameter(description = "Exact match names only (if true, partial name matches are excluded)") @QueryParam("exactmatchnames") Boolean exactMatchNamesOnly,
@Parameter(description = "Default resources (without identifiers) only") @QueryParam("default") Boolean defaultResource,
@ -212,7 +220,7 @@ public class ArbitraryResource {
}
List<ArbitraryResourceData> resources = repository.getArbitraryRepository()
.searchArbitraryResources(service, query, identifier, names, title, description, usePrefixOnly,
.searchArbitraryResources(service, query, identifier, names, title, description, keywords, usePrefixOnly,
exactMatchNames, defaultRes, mode, minLevel, followedOnly, excludeBlocked, includeMetadata, includeStatus,
before, after, limit, offset, reverse);
@ -1185,6 +1193,90 @@ public class ArbitraryResource {
}
}
@GET
@Path("/indices")
@Operation(
summary = "Find matching arbitrary resource indices",
description = "",
responses = {
@ApiResponse(
description = "indices",
content = @Content(
array = @ArraySchema(
schema = @Schema(
implementation = ArbitraryDataIndexScorecard.class
)
)
)
)
}
)
public List<ArbitraryDataIndexScorecard> searchIndices(@QueryParam("terms") String[] terms) {
List<ArbitraryDataIndexDetail> indices = new ArrayList<>();
// get index details for each term
for( String term : terms ) {
List<ArbitraryDataIndexDetail> details = IndexCache.getInstance().getIndicesByTerm().get(term);
if( details != null ) {
indices.addAll(details);
}
}
// sum up the scores for each index with identical attributes
Map<ArbitraryDataIndexScoreKey, Double> scoreForKey
= indices.stream()
.collect(
Collectors.groupingBy(
index -> new ArbitraryDataIndexScoreKey(index.name, index.category, index.link),
Collectors.summingDouble(detail -> 1.0 / detail.rank)
)
);
// create scorecards for each index group and put them in descending order by score
List<ArbitraryDataIndexScorecard> scorecards
= scoreForKey.entrySet().stream().map(
entry
->
new ArbitraryDataIndexScorecard(
entry.getValue(),
entry.getKey().name,
entry.getKey().category,
entry.getKey().link)
)
.sorted(Comparator.comparingDouble(ArbitraryDataIndexScorecard::getScore).reversed())
.collect(Collectors.toList());
return scorecards;
}
@GET
@Path("/indices/{name}/{idPrefix}")
@Operation(
summary = "Find matching arbitrary resource indices for a registered name and identifier prefix",
description = "",
responses = {
@ApiResponse(
description = "indices",
content = @Content(
array = @ArraySchema(
schema = @Schema(
implementation = ArbitraryDataIndexDetail.class
)
)
)
)
}
)
public List<ArbitraryDataIndexDetail> searchIndicesByName(@PathParam("name") String name, @PathParam("idPrefix") String idPrefix) {
return
IndexCache.getInstance().getIndicesByIssuer()
.getOrDefault(name, new ArrayList<>(0)).stream()
.filter( indexDetail -> indexDetail.indexIdentifer.startsWith(idPrefix))
.collect(Collectors.toList());
}
// Shared methods

141
src/main/java/org/qortal/api/resource/CrossChainResource.java
View File

@ -10,11 +10,13 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.glassfish.jersey.media.multipart.ContentDisposition;
import org.qortal.api.ApiError;
import org.qortal.api.ApiErrors;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.Security;
import org.qortal.api.model.CrossChainCancelRequest;
import org.qortal.api.model.CrossChainTradeLedgerEntry;
import org.qortal.api.model.CrossChainTradeSummary;
import org.qortal.controller.tradebot.TradeBot;
import org.qortal.crosschain.ACCT;
@ -44,10 +46,14 @@ import org.qortal.utils.Base58;
import org.qortal.utils.ByteArray;
import org.qortal.utils.NTP;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import java.io.IOException;
import java.util.*;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@ -61,6 +67,13 @@ public class CrossChainResource {
@Context
HttpServletRequest request;
@Context
HttpServletResponse response;
@Context
ServletContext context;
@GET
@Path("/tradeoffers")
@Operation(
@ -258,11 +271,11 @@ public class CrossChainResource {
example = "1597310000000"
) @QueryParam("minimumTimestamp") Long minimumTimestamp,
@Parameter(
description = "Optionally filter by buyer Qortal address"
) @QueryParam("buyerAddress") String buyerAddress,
description = "Optionally filter by buyer Qortal public key"
) @QueryParam("buyerPublicKey") String buyerPublicKey58,
@Parameter(
description = "Optionally filter by seller Qortal address"
) @QueryParam("sellerAddress") String sellerAddress,
description = "Optionally filter by seller Qortal public key"
) @QueryParam("sellerPublicKey") String sellerPublicKey58,
@Parameter( ref = "limit") @QueryParam("limit") Integer limit,
@Parameter( ref = "offset" ) @QueryParam("offset") Integer offset,
@Parameter( ref = "reverse" ) @QueryParam("reverse") Boolean reverse) {
@ -274,6 +287,10 @@ public class CrossChainResource {
if (minimumTimestamp != null && minimumTimestamp <= 0)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
// Decode public keys
byte[] buyerPublicKey = decodePublicKey(buyerPublicKey58);
byte[] sellerPublicKey = decodePublicKey(sellerPublicKey58);
final Boolean isFinished = Boolean.TRUE;
try (final Repository repository = RepositoryManager.getRepository()) {
@ -304,7 +321,7 @@ public class CrossChainResource {
byte[] codeHash = acctInfo.getKey().value;
ACCT acct = acctInfo.getValue().get();
List<ATStateData> atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, buyerAddress, sellerAddress,
List<ATStateData> atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, buyerPublicKey, sellerPublicKey,
isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumFinalHeight,
limit, offset, reverse);
@ -343,6 +360,120 @@ public class CrossChainResource {
}
}
/**
* Decode Public Key
*
* @param publicKey58 the public key in a string
*
* @return the public key in bytes
*/
private byte[] decodePublicKey(String publicKey58) {
if( publicKey58 == null ) return null;
if( publicKey58.isEmpty() ) return new byte[0];
byte[] publicKey;
try {
publicKey = Base58.decode(publicKey58);
} catch (NumberFormatException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY, e);
}
// Correct size for public key?
if (publicKey.length != Transformer.PUBLIC_KEY_LENGTH)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY);
return publicKey;
}
@GET
@Path("/ledger/{publicKey}")
@Operation(
summary = "Accounting entries for all trades.",
description = "Returns accounting entries for all completed cross-chain trades",
responses = {
@ApiResponse(
content = @Content(
schema = @Schema(
type = "string",
format = "byte"
)
)
)
}
)
@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
public HttpServletResponse getLedgerEntries(
@PathParam("publicKey") String publicKey58,
@Parameter(
description = "Only return trades that completed on/after this timestamp (milliseconds since epoch)",
example = "1597310000000"
) @QueryParam("minimumTimestamp") Long minimumTimestamp) {
byte[] publicKey = decodePublicKey(publicKey58);
// minimumTimestamp (if given) needs to be positive
if (minimumTimestamp != null && minimumTimestamp <= 0)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
try (final Repository repository = RepositoryManager.getRepository()) {
Integer minimumFinalHeight = null;
if (minimumTimestamp != null) {
minimumFinalHeight = repository.getBlockRepository().getHeightFromTimestamp(minimumTimestamp);
// If not found in the block repository it will return either 0 or 1
if (minimumFinalHeight == 0 || minimumFinalHeight == 1) {
// Try the archive
minimumFinalHeight = repository.getBlockArchiveRepository().getHeightFromTimestamp(minimumTimestamp);
}
if (minimumFinalHeight == 0)
// We don't have any blocks since minimumTimestamp, let alone trades, so nothing to return
return response;
// height returned from repository is for block BEFORE timestamp
// but we want trades AFTER timestamp so bump height accordingly
minimumFinalHeight++;
}
List<CrossChainTradeLedgerEntry> crossChainTradeLedgerEntries = new ArrayList<>();
Map<ByteArray, Supplier<ACCT>> acctsByCodeHash = SupportedBlockchain.getAcctMap();
// collect ledger entries for each ACCT
for (Map.Entry<ByteArray, Supplier<ACCT>> acctInfo : acctsByCodeHash.entrySet()) {
byte[] codeHash = acctInfo.getKey().value;
ACCT acct = acctInfo.getValue().get();
// collect buys and sells
CrossChainUtils.collectLedgerEntries(publicKey, repository, minimumFinalHeight, crossChainTradeLedgerEntries, codeHash, acct, true);
CrossChainUtils.collectLedgerEntries(publicKey, repository, minimumFinalHeight, crossChainTradeLedgerEntries, codeHash, acct, false);
}
crossChainTradeLedgerEntries.sort((a, b) -> Longs.compare(a.getTradeTimestamp(), b.getTradeTimestamp()));
response.setStatus(HttpServletResponse.SC_OK);
response.setContentType("text/csv");
response.setHeader(
HttpHeaders.CONTENT_DISPOSITION,
ContentDisposition
.type("attachment")
.fileName(CrossChainUtils.createLedgerFileName(Crypto.toAddress(publicKey)))
.build()
.toString()
);
CrossChainUtils.writeToLedger( response.getWriter(), crossChainTradeLedgerEntries);
return response;
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
} catch (IOException e) {
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
return response;
}
}
@GET
@Path("/price/{blockchain}")
@Operation(

139
src/main/java/org/qortal/api/resource/CrossChainUtils.java
View File

@ -10,21 +10,36 @@ import org.bitcoinj.script.ScriptBuilder;
import org.bouncycastle.util.Strings;
import org.json.simple.JSONObject;
import org.qortal.api.model.CrossChainTradeLedgerEntry;
import org.qortal.api.model.crosschain.BitcoinyTBDRequest;
import org.qortal.crosschain.*;
import org.qortal.data.at.ATData;
import org.qortal.data.at.ATStateData;
import org.qortal.data.crosschain.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.utils.Amounts;
import org.qortal.utils.BitTwiddling;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.stream.Collectors;
public class CrossChainUtils {
public static final String QORT_CURRENCY_CODE = "QORT";
private static final Logger LOGGER = LogManager.getLogger(CrossChainUtils.class);
public static final String CORE_API_CALL = "Core API Call";
public static final String QORTAL_EXCHANGE_LABEL = "Qortal";
public static ServerConfigurationInfo buildServerConfigurationInfo(Bitcoiny blockchain) {
@ -632,4 +647,128 @@ public class CrossChainUtils {
byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA);
return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes);
}
/**
* Write To Ledger
*
* @param writer the writer to the ledger
* @param entries the entries to write to the ledger
*
* @throws IOException
*/
public static void writeToLedger(Writer writer, List<CrossChainTradeLedgerEntry> entries) throws IOException {
BufferedWriter bufferedWriter = new BufferedWriter(writer);
StringJoiner header = new StringJoiner(",");
header.add("Market");
header.add("Currency");
header.add("Quantity");
header.add("Commission Paid");
header.add("Commission Currency");
header.add("Total Price");
header.add("Date Time");
header.add("Exchange");
bufferedWriter.append(header.toString());
DateFormat dateFormatter = new SimpleDateFormat("yyyyMMdd HH:mm");
dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
for( CrossChainTradeLedgerEntry entry : entries ) {
StringJoiner joiner = new StringJoiner(",");
joiner.add(entry.getMarket());
joiner.add(entry.getCurrency());
joiner.add(String.valueOf(Amounts.prettyAmount(entry.getQuantity())));
joiner.add(String.valueOf(Amounts.prettyAmount(entry.getFeeAmount())));
joiner.add(entry.getFeeCurrency());
joiner.add(String.valueOf(Amounts.prettyAmount(entry.getTotalPrice())));
joiner.add(dateFormatter.format(new Date(entry.getTradeTimestamp())));
joiner.add(QORTAL_EXCHANGE_LABEL);
bufferedWriter.newLine();
bufferedWriter.append(joiner.toString());
}
bufferedWriter.newLine();
bufferedWriter.flush();
}
/**
* Create Ledger File Name
*
* Create a file name the includes timestamp and address.
*
* @param address the address
*
* @return the file name created
*/
public static String createLedgerFileName(String address) {
DateFormat dateFormatter = new SimpleDateFormat("yyyyMMddHHmmss");
String fileName = "ledger-" + address + "-" + dateFormatter.format(new Date());
return fileName;
}
/**
* Collect Ledger Entries
*
* @param publicKey the public key for the ledger entries, buy and sell
* @param repository the data repository
* @param minimumFinalHeight the minimum block height for entries to be collected
* @param entries the ledger entries to add to
* @param codeHash code hash for the entry blockchain
* @param acct the ACCT for the entry blockchain
* @param isBuy true collecting entries for a buy, otherwise false
*
* @throws DataException
*/
public static void collectLedgerEntries(
byte[] publicKey,
Repository repository,
Integer minimumFinalHeight,
List<CrossChainTradeLedgerEntry> entries,
byte[] codeHash,
ACCT acct,
boolean isBuy) throws DataException {
// get all the final AT states for the code hash (foreign coin)
List<ATStateData> atStates
= repository.getATRepository().getMatchingFinalATStates(
codeHash,
isBuy ? publicKey : null,
!isBuy ? publicKey : null,
Boolean.TRUE, acct.getModeByteOffset(),
(long) AcctMode.REDEEMED.value,
minimumFinalHeight,
null, null, false
);
String foreignBlockchainCurrencyCode = acct.getBlockchain().getCurrencyCode();
// for each trade, build ledger entry, collect ledger entry
for (ATStateData atState : atStates) {
CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState);
// We also need block timestamp for use as trade timestamp
long localTimestamp = repository.getBlockRepository().getTimestampFromHeight(atState.getHeight());
if (localTimestamp == 0) {
// Try the archive
localTimestamp = repository.getBlockArchiveRepository().getTimestampFromHeight(atState.getHeight());
}
CrossChainTradeLedgerEntry ledgerEntry
= new CrossChainTradeLedgerEntry(
isBuy ? QORT_CURRENCY_CODE : foreignBlockchainCurrencyCode,
isBuy ? foreignBlockchainCurrencyCode : QORT_CURRENCY_CODE,
isBuy ? crossChainTradeData.qortAmount : crossChainTradeData.expectedForeignAmount,
0,
foreignBlockchainCurrencyCode,
isBuy ? crossChainTradeData.expectedForeignAmount : crossChainTradeData.qortAmount,
localTimestamp);
entries.add(ledgerEntry);
}
}
}

102
src/main/java/org/qortal/api/websocket/DataMonitorSocket.java Normal file
View File

@ -0,0 +1,102 @@
package org.qortal.api.websocket;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.eclipse.jetty.websocket.api.Session;
import org.eclipse.jetty.websocket.api.WebSocketException;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage;
import org.eclipse.jetty.websocket.api.annotations.WebSocket;
import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
import org.qortal.api.ApiError;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.DataMonitorInfo;
import org.qortal.event.DataMonitorEvent;
import org.qortal.event.Event;
import org.qortal.event.EventBus;
import org.qortal.event.Listener;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.utils.Base58;
import java.io.IOException;
import java.io.StringWriter;
import java.util.List;
@WebSocket
@SuppressWarnings("serial")
public class DataMonitorSocket extends ApiWebSocket implements Listener {
private static final Logger LOGGER = LogManager.getLogger(DataMonitorSocket.class);
@Override
public void configure(WebSocketServletFactory factory) {
LOGGER.info("configure");
factory.register(DataMonitorSocket.class);
EventBus.INSTANCE.addListener(this);
}
@Override
public void listen(Event event) {
if (!(event instanceof DataMonitorEvent))
return;
DataMonitorEvent dataMonitorEvent = (DataMonitorEvent) event;
for (Session session : getSessions())
sendDataEventSummary(session, buildInfo(dataMonitorEvent));
}
private DataMonitorInfo buildInfo(DataMonitorEvent dataMonitorEvent) {
return new DataMonitorInfo(
dataMonitorEvent.getTimestamp(),
dataMonitorEvent.getIdentifier(),
dataMonitorEvent.getName(),
dataMonitorEvent.getService(),
dataMonitorEvent.getDescription(),
dataMonitorEvent.getTransactionTimestamp(),
dataMonitorEvent.getLatestPutTimestamp()
);
}
@OnWebSocketConnect
@Override
public void onWebSocketConnect(Session session) {
super.onWebSocketConnect(session);
}
@OnWebSocketClose
@Override
public void onWebSocketClose(Session session, int statusCode, String reason) {
super.onWebSocketClose(session, statusCode, reason);
}
@OnWebSocketError
public void onWebSocketError(Session session, Throwable throwable) {
/* We ignore errors for now, but method here to silence log spam */
}
@OnWebSocketMessage
public void onWebSocketMessage(Session session, String message) {
LOGGER.info("onWebSocketMessage: message = " + message);
}
private void sendDataEventSummary(Session session, DataMonitorInfo dataMonitorInfo) {
StringWriter stringWriter = new StringWriter();
try {
marshall(stringWriter, dataMonitorInfo);
session.getRemote().sendStringByFuture(stringWriter.toString());
} catch (IOException | WebSocketException e) {
// No output this time
}
}
}

10
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@ -439,7 +439,15 @@ public class ArbitraryDataReader {
// Ensure the complete hash matches the joined chunks
if (!Arrays.equals(arbitraryDataFile.digest(), transactionData.getData())) {
// Delete the invalid file
arbitraryDataFile.delete();
LOGGER.info("Deleting invalid file: path = " + arbitraryDataFile.getFilePath());
if( arbitraryDataFile.delete() ) {
LOGGER.info("Deleted invalid file successfully: path = " + arbitraryDataFile.getFilePath());
}
else {
LOGGER.warn("Could not delete invalid file: path = " + arbitraryDataFile.getFilePath());
}
throw new DataException("Unable to validate complete file hash");
}
}

23
src/main/java/org/qortal/block/Block.java
View File

@ -39,6 +39,7 @@ import org.qortal.transform.block.BlockTransformer;
import org.qortal.transform.transaction.TransactionTransformer;
import org.qortal.utils.Amounts;
import org.qortal.utils.Base58;
import org.qortal.utils.Groups;
import org.qortal.utils.NTP;
import java.io.ByteArrayOutputStream;
@ -150,7 +151,7 @@ public class Block {
final BlockChain blockChain = BlockChain.getInstance();
ExpandedAccount(Repository repository, RewardShareData rewardShareData) throws DataException {
ExpandedAccount(Repository repository, RewardShareData rewardShareData, int blockHeight) throws DataException {
this.rewardShareData = rewardShareData;
this.sharePercent = this.rewardShareData.getSharePercent();
@ -159,7 +160,12 @@ public class Block {
this.isMinterFounder = Account.isFounder(mintingAccountData.getFlags());
this.isRecipientAlsoMinter = this.rewardShareData.getRecipient().equals(this.mintingAccount.getAddress());
this.isMinterMember = repository.getGroupRepository().memberExists(BlockChain.getInstance().getMintingGroupId(), this.mintingAccount.getAddress());
this.isMinterMember
= Groups.memberExistsInAnyGroup(
repository.getGroupRepository(),
Groups.getGroupIdsToMint(BlockChain.getInstance(), blockHeight),
this.mintingAccount.getAddress()
);
if (this.isRecipientAlsoMinter) {
// Self-share: minter is also recipient
@ -435,9 +441,9 @@ public class Block {
if (height >= BlockChain.getInstance().getGroupMemberCheckHeight()) {
onlineAccounts.removeIf(a -> {
try {
int groupId = BlockChain.getInstance().getMintingGroupId();
List<Integer> groupIdsToMint = Groups.getGroupIdsToMint(BlockChain.getInstance(), height);
String address = Account.getRewardShareMintingAddress(repository, a.getPublicKey());
boolean isMinterGroupMember = repository.getGroupRepository().memberExists(groupId, address);
boolean isMinterGroupMember = Groups.memberExistsInAnyGroup(repository.getGroupRepository(), groupIdsToMint, address);
return !isMinterGroupMember;
} catch (DataException e) {
// Something went wrong, so remove the account
@ -753,7 +759,7 @@ public class Block {
List<ExpandedAccount> expandedAccounts = new ArrayList<>();
for (RewardShareData rewardShare : this.cachedOnlineRewardShares) {
expandedAccounts.add(new ExpandedAccount(repository, rewardShare));
expandedAccounts.add(new ExpandedAccount(repository, rewardShare, this.blockData.getHeight()));
}
this.cachedExpandedAccounts = expandedAccounts;
@ -2485,11 +2491,10 @@ public class Block {
try (final Repository repository = RepositoryManager.getRepository()) {
GroupRepository groupRepository = repository.getGroupRepository();
List<Integer> mintingGroupIds = Groups.getGroupIdsToMint(BlockChain.getInstance(), this.blockData.getHeight());
// all minter admins
List<String> minterAdmins
= groupRepository.getGroupAdmins(BlockChain.getInstance().getMintingGroupId()).stream()
.map(GroupAdminData::getAdmin)
.collect(Collectors.toList());
List<String> minterAdmins = Groups.getAllAdmins(groupRepository, mintingGroupIds);
// all minter admins that are online
List<ExpandedAccount> onlineMinterAdminAccounts

12
src/main/java/org/qortal/block/BlockChain.java
View File

@ -212,7 +212,13 @@ public class BlockChain {
private int minAccountLevelToRewardShare;
private int maxRewardSharesPerFounderMintingAccount;
private int founderEffectiveMintingLevel;
private int mintingGroupId;
public static class IdsForHeight {
public int height;
public List<Integer> ids;
}
private List<IdsForHeight> mintingGroupIds;
/** Minimum time to retain online account signatures (ms) for block validity checks. */
private long onlineAccountSignaturesMinLifetime;
@ -544,8 +550,8 @@ public class BlockChain {
return this.onlineAccountSignaturesMaxLifetime;
}
public int getMintingGroupId() {
return this.mintingGroupId;
public List<IdsForHeight> getMintingGroupIds() {
return mintingGroupIds;
}
public CiyamAtSettings getCiyamAtSettings() {

16
src/main/java/org/qortal/controller/Controller.java
View File

@ -423,6 +423,12 @@ public class Controller extends Thread {
LOGGER.info("Db Cache Disabled");
}
LOGGER.info("Arbitrary Indexing Starting ...");
ArbitraryIndexUtils.startCaching(
Settings.getInstance().getArbitraryIndexingPriority(),
Settings.getInstance().getArbitraryIndexingFrequency()
);
if( Settings.getInstance().isBalanceRecorderEnabled() ) {
Optional<HSQLDBBalanceRecorder> recorder = HSQLDBBalanceRecorder.getInstance();
@ -541,6 +547,16 @@ public class Controller extends Thread {
ArbitraryDataStorageManager.getInstance().start();
ArbitraryDataRenderManager.getInstance().start();
// start rebuild arbitrary resource cache timer task
if( Settings.getInstance().isRebuildArbitraryResourceCacheTaskEnabled() ) {
new Timer().schedule(
new RebuildArbitraryResourceCacheTask(),
Settings.getInstance().getRebuildArbitraryResourceCacheTaskDelay() * RebuildArbitraryResourceCacheTask.MILLIS_IN_MINUTE,
Settings.getInstance().getRebuildArbitraryResourceCacheTaskPeriod() * RebuildArbitraryResourceCacheTask.MILLIS_IN_HOUR
);
}
LOGGER.info("Starting online accounts manager");
OnlineAccountsManager.getInstance().start();

12
src/main/java/org/qortal/controller/OnlineAccountsManager.java
View File

@ -25,6 +25,7 @@ import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
import org.qortal.utils.Groups;
import org.qortal.utils.NTP;
import org.qortal.utils.NamedThreadFactory;
@ -225,11 +226,14 @@ public class OnlineAccountsManager {
Set<OnlineAccountData> onlineAccountsToAdd = new HashSet<>();
Set<OnlineAccountData> onlineAccountsToRemove = new HashSet<>();
try (final Repository repository = RepositoryManager.getRepository()) {
int blockHeight = repository.getBlockRepository().getBlockchainHeight();
List<String> mintingGroupMemberAddresses
= repository.getGroupRepository()
.getGroupMembers(BlockChain.getInstance().getMintingGroupId()).stream()
.map(GroupMemberData::getMember)
.collect(Collectors.toList());
= Groups.getAllMembers(
repository.getGroupRepository(),
Groups.getGroupIdsToMint(BlockChain.getInstance(), blockHeight)
);
for (OnlineAccountData onlineAccountData : this.onlineAccountsImportQueue) {
if (isStopping)

167
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java
View File

@ -2,22 +2,30 @@ package org.qortal.controller.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.event.DataMonitorEvent;
import org.qortal.event.EventBus;
import org.qortal.gui.SplashFrame;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.Transaction;
import org.qortal.utils.Base58;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
public class ArbitraryDataCacheManager extends Thread {
@ -29,6 +37,11 @@ public class ArbitraryDataCacheManager extends Thread {
/** Queue of arbitrary transactions that require cache updates */
private final List<ArbitraryTransactionData> updateQueue = Collections.synchronizedList(new ArrayList<>());
private static final NumberFormat FORMATTER = NumberFormat.getNumberInstance();
static {
FORMATTER.setGroupingUsed(true);
}
public static synchronized ArbitraryDataCacheManager getInstance() {
if (instance == null) {
@ -45,17 +58,22 @@ public class ArbitraryDataCacheManager extends Thread {
try {
while (!Controller.isStopping()) {
Thread.sleep(500L);
try {
Thread.sleep(500L);
// Process queue
processResourceQueue();
// Process queue
processResourceQueue();
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
Thread.sleep(600_000L); // wait 10 minutes to continue
}
}
} catch (InterruptedException e) {
// Fall through to exit thread
}
// Clear queue before terminating thread
processResourceQueue();
// Clear queue before terminating thread
processResourceQueue();
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
public void shutdown() {
@ -85,14 +103,25 @@ public class ArbitraryDataCacheManager extends Thread {
// Update arbitrary resource caches
try {
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceCache(repository);
arbitraryTransaction.updateArbitraryMetadataCache(repository);
arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, new HashSet<>(0), new HashMap<>(0));
repository.saveChanges();
// Update status as separate commit, as this is more prone to failure
arbitraryTransaction.updateArbitraryResourceStatus(repository);
repository.saveChanges();
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
transactionData.getIdentifier(),
transactionData.getName(),
transactionData.getService().name(),
"updated resource cache and status, queue",
transactionData.getTimestamp(),
transactionData.getTimestamp()
)
);
LOGGER.debug(() -> String.format("Finished processing transaction %.8s in arbitrary resource queue...", Base58.encode(transactionData.getSignature())));
} catch (DataException e) {
@ -103,6 +132,9 @@ public class ArbitraryDataCacheManager extends Thread {
} catch (DataException e) {
LOGGER.error("Repository issue while processing arbitrary resource cache updates", e);
}
catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
public void addToUpdateQueue(ArbitraryTransactionData transactionData) {
@ -148,34 +180,66 @@ public class ArbitraryDataCacheManager extends Thread {
LOGGER.info("Building arbitrary resources cache...");
SplashFrame.getInstance().updateStatus("Building QDN cache - please wait...");
final int batchSize = 100;
final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
int offset = 0;
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository().getLatestArbitraryTransactions();
LOGGER.info("arbitrary transactions: count = " + allArbitraryTransactionsInDescendingOrder.size());
List<ArbitraryResourceData> resources = repository.getArbitraryRepository().getArbitraryResources(null, null, true);
Map<ArbitraryTransactionDataHashWrapper, ArbitraryResourceData> resourceByWrapper = new HashMap<>(resources.size());
for( ArbitraryResourceData resource : resources ) {
resourceByWrapper.put(
new ArbitraryTransactionDataHashWrapper(resource.service.value, resource.name, resource.identifier),
resource
);
}
LOGGER.info("arbitrary resources: count = " + resourceByWrapper.size());
Set<ArbitraryTransactionDataHashWrapper> latestTransactionsWrapped = new HashSet<>(allArbitraryTransactionsInDescendingOrder.size());
// Loop through all ARBITRARY transactions, and determine latest state
while (!Controller.isStopping()) {
LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1);
LOGGER.info(
"Fetching arbitrary transactions {} - {} / {} Total",
FORMATTER.format(offset),
FORMATTER.format(offset+batchSize-1),
FORMATTER.format(allArbitraryTransactionsInDescendingOrder.size())
);
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false);
if (signatures.isEmpty()) {
List<ArbitraryTransactionData> transactionsToProcess
= allArbitraryTransactionsInDescendingOrder.stream()
.skip(offset)
.limit(batchSize)
.collect(Collectors.toList());
if (transactionsToProcess.isEmpty()) {
// Complete
break;
}
// Expand signatures to transactions
for (byte[] signature : signatures) {
ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository
.getTransactionRepository().fromSignature(signature);
try {
for( ArbitraryTransactionData transactionData : transactionsToProcess) {
if (transactionData.getService() == null) {
// Unsupported service - ignore this resource
continue;
}
if (transactionData.getService() == null) {
// Unsupported service - ignore this resource
continue;
latestTransactionsWrapped.add(new ArbitraryTransactionDataHashWrapper(transactionData));
// Update arbitrary resource caches
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, latestTransactionsWrapped, resourceByWrapper);
}
// Update arbitrary resource caches
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceCache(repository);
arbitraryTransaction.updateArbitraryMetadataCache(repository);
repository.saveChanges();
} catch (DataException e) {
repository.discardChanges();
LOGGER.error(e.getMessage(), e);
}
offset += batchSize;
}
@ -193,6 +257,11 @@ public class ArbitraryDataCacheManager extends Thread {
repository.discardChanges();
throw new DataException("Build of arbitrary resources cache failed.");
}
catch (Exception e) {
LOGGER.error(e.getMessage(), e);
return false;
}
}
private boolean refreshArbitraryStatuses(Repository repository) throws DataException {
@ -200,27 +269,48 @@ public class ArbitraryDataCacheManager extends Thread {
LOGGER.info("Refreshing arbitrary resource statuses for locally hosted transactions...");
SplashFrame.getInstance().updateStatus("Refreshing statuses - please wait...");
final int batchSize = 100;
final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
int offset = 0;
List<ArbitraryTransactionData> allHostedTransactions
= ArbitraryDataStorageManager.getInstance()
.listAllHostedTransactions(repository, null, null);
// Loop through all ARBITRARY transactions, and determine latest state
while (!Controller.isStopping()) {
LOGGER.info("Fetching hosted transactions {} - {}", offset, offset+batchSize-1);
LOGGER.info(
"Fetching hosted transactions {} - {} / {} Total",
FORMATTER.format(offset),
FORMATTER.format(offset+batchSize-1),
FORMATTER.format(allHostedTransactions.size())
);
List<ArbitraryTransactionData> hostedTransactions
= allHostedTransactions.stream()
.skip(offset)
.limit(batchSize)
.collect(Collectors.toList());
List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, batchSize, offset);
if (hostedTransactions.isEmpty()) {
// Complete
break;
}
// Loop through hosted transactions
for (ArbitraryTransactionData transactionData : hostedTransactions) {
try {
// Loop through hosted transactions
for (ArbitraryTransactionData transactionData : hostedTransactions) {
// Determine status and update cache
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceStatus(repository);
// Determine status and update cache
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceStatus(repository);
}
repository.saveChanges();
} catch (DataException e) {
repository.discardChanges();
LOGGER.error(e.getMessage(), e);
}
offset += batchSize;
}
@ -234,6 +324,11 @@ public class ArbitraryDataCacheManager extends Thread {
repository.discardChanges();
throw new DataException("Refresh of arbitrary resource statuses failed.");
}
catch (Exception e) {
LOGGER.error(e.getMessage(), e);
return false;
}
}
}

160
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
View File

@ -2,9 +2,10 @@ package org.qortal.controller.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.event.DataMonitorEvent;
import org.qortal.event.EventBus;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
@ -21,8 +22,12 @@ import java.nio.file.Paths;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD;
@ -77,6 +82,19 @@ public class ArbitraryDataCleanupManager extends Thread {
final int limit = 100;
int offset = 0;
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
try (final Repository repository = RepositoryManager.getRepository()) {
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactions();
} catch( Exception e) {
LOGGER.error(e.getMessage(), e);
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
}
Set<ArbitraryTransactionData> processedTransactions = new HashSet<>();
try {
while (!isStopping) {
Thread.sleep(30000);
@ -107,27 +125,31 @@ public class ArbitraryDataCleanupManager extends Thread {
// Any arbitrary transactions we want to fetch data for?
try (final Repository repository = RepositoryManager.getRepository()) {
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true);
// LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
List<ArbitraryTransactionData> transactions = allArbitraryTransactionsInDescendingOrder.stream().skip(offset).limit(limit).collect(Collectors.toList());
if (isStopping) {
return;
}
if (signatures == null || signatures.isEmpty()) {
if (transactions == null || transactions.isEmpty()) {
offset = 0;
continue;
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactions();
transactions = allArbitraryTransactionsInDescendingOrder.stream().limit(limit).collect(Collectors.toList());
processedTransactions.clear();
}
offset += limit;
now = NTP.getTime();
// Loop through the signatures in this batch
for (int i=0; i<signatures.size(); i++) {
for (int i=0; i<transactions.size(); i++) {
if (isStopping) {
return;
}
byte[] signature = signatures.get(i);
if (signature == null) {
ArbitraryTransactionData arbitraryTransactionData = transactions.get(i);
if (arbitraryTransactionData == null) {
continue;
}
@ -136,9 +158,7 @@ public class ArbitraryDataCleanupManager extends Thread {
Thread.sleep(5000);
}
// Fetch the transaction data
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
if (arbitraryTransactionData == null || arbitraryTransactionData.getService() == null) {
if (arbitraryTransactionData.getService() == null) {
continue;
}
@ -147,6 +167,8 @@ public class ArbitraryDataCleanupManager extends Thread {
continue;
}
boolean mostRecentTransaction = processedTransactions.add(arbitraryTransactionData);
// Check if we have the complete file
boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData);
@ -167,20 +189,54 @@ public class ArbitraryDataCleanupManager extends Thread {
LOGGER.info("Deleting transaction {} because we can't host its data",
Base58.encode(arbitraryTransactionData.getSignature()));
ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"can't store data, deleting",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
continue;
}
// Check to see if we have had a more recent PUT
boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
if (hasMoreRecentPutTransaction) {
if (!mostRecentTransaction) {
// There is a more recent PUT transaction than the one we are currently processing.
// When a PUT is issued, it replaces any layers that would have been there before.
// Therefore any data relating to this older transaction is no longer needed.
LOGGER.info(String.format("Newer PUT found for %s %s since transaction %s. " +
"Deleting all files associated with the earlier transaction.", arbitraryTransactionData.getService(),
arbitraryTransactionData.getName(), Base58.encode(signature)));
arbitraryTransactionData.getName(), Base58.encode(arbitraryTransactionData.getSignature())));
ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
Optional<ArbitraryTransactionData> moreRecentPutTransaction
= processedTransactions.stream()
.filter(data -> data.equals(arbitraryTransactionData))
.findAny();
if( moreRecentPutTransaction.isPresent() ) {
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"deleting data due to replacement",
arbitraryTransactionData.getTimestamp(),
moreRecentPutTransaction.get().getTimestamp()
)
);
}
else {
LOGGER.warn("Something went wrong with the most recent put transaction determination!");
}
continue;
}
@ -199,7 +255,21 @@ public class ArbitraryDataCleanupManager extends Thread {
LOGGER.debug(String.format("Transaction %s has complete file and all chunks",
Base58.encode(arbitraryTransactionData.getSignature())));
ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
boolean wasDeleted = ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
if( wasDeleted ) {
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"deleting file, retaining chunks",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
}
continue;
}
@ -237,17 +307,6 @@ public class ArbitraryDataCleanupManager extends Thread {
this.storageLimitReached(repository);
}
// Delete random data associated with name if we're over our storage limit for this name
// Use the DELETION_THRESHOLD, for the same reasons as above
for (String followedName : ListUtils.followedNames()) {
if (isStopping) {
return;
}
if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) {
this.storageLimitReachedForName(repository, followedName);
}
}
} catch (DataException e) {
LOGGER.error("Repository issue when cleaning up arbitrary transaction data", e);
}
@ -326,25 +385,6 @@ public class ArbitraryDataCleanupManager extends Thread {
// FUTURE: consider reducing the expiry time of the reader cache
}
public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException {
// We think that the storage limit has been reached for supplied name - but we should double check
if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, DELETION_THRESHOLD)) {
// We have space available for this name, so don't delete anything
return;
}
// Delete a batch of random chunks associated with this name
// This reduces the chance of too many nodes deleting the same chunk
// when they reach their storage limit
Path dataPath = Paths.get(Settings.getInstance().getDataPath());
for (int i=0; i<CHUNK_DELETION_BATCH_SIZE; i++) {
if (isStopping) {
return;
}
this.deleteRandomFile(repository, dataPath.toFile(), name);
}
}
/**
* Iteratively walk through given directory and delete a single random file
*
@ -423,6 +463,7 @@ public class ArbitraryDataCleanupManager extends Thread {
}
LOGGER.info("Deleting random file {} because we have reached max storage capacity...", randomItem.toString());
fireRandomItemDeletionNotification(randomItem, repository, "Deleting random file, because we have reached max storage capacity");
boolean success = randomItem.delete();
if (success) {
try {
@ -437,6 +478,35 @@ public class ArbitraryDataCleanupManager extends Thread {
return false;
}
private void fireRandomItemDeletionNotification(File randomItem, Repository repository, String reason) {
try {
Path parentFileNamePath = randomItem.toPath().toAbsolutePath().getParent().getFileName();
if (parentFileNamePath != null) {
String signature58 = parentFileNamePath.toString();
byte[] signature = Base58.decode(signature58);
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData != null && transactionData.getType() == Transaction.TransactionType.ARBITRARY) {
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
reason,
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
}
}
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
private void cleanupTempDirectory(String folder, long now, long minAge) {
String baseDir = Settings.getInstance().getTempDataPath();
Path tempDir = Paths.get(baseDir, folder);

21
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataExamination.java Normal file
View File

@ -0,0 +1,21 @@
package org.qortal.controller.arbitrary;
public class ArbitraryDataExamination {
private boolean pass;
private String notes;
public ArbitraryDataExamination(boolean pass, String notes) {
this.pass = pass;
this.notes = notes;
}
public boolean isPass() {
return pass;
}
public String getNotes() {
return notes;
}
}

2
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java
View File

@ -5,6 +5,8 @@ import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.event.DataMonitorEvent;
import org.qortal.event.EventBus;
import org.qortal.network.Peer;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;

180
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@ -10,6 +10,8 @@ import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.event.DataMonitorEvent;
import org.qortal.event.EventBus;
import org.qortal.network.Network;
import org.qortal.network.Peer;
import org.qortal.repository.DataException;
@ -28,6 +30,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
public class ArbitraryDataManager extends Thread {
@ -195,13 +198,35 @@ public class ArbitraryDataManager extends Thread {
final int limit = 100;
int offset = 0;
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
try (final Repository repository = RepositoryManager.getRepository()) {
if( name == null ) {
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactions();
}
else {
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactionsByName(name);
}
} catch( Exception e) {
LOGGER.error(e.getMessage(), e);
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
}
// collect processed transactions in a set to ensure outdated data transactions do not get fetched
Set<ArbitraryTransactionDataHashWrapper> processedTransactions = new HashSet<>();
while (!isStopping) {
Thread.sleep(1000L);
// Any arbitrary transactions we want to fetch data for?
try (final Repository repository = RepositoryManager.getRepository()) {
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, name, null, ConfirmationStatus.BOTH, limit, offset, true);
// LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
List<byte[]> signatures = processTransactionsForSignatures(limit, offset, allArbitraryTransactionsInDescendingOrder, processedTransactions);
if (signatures == null || signatures.isEmpty()) {
offset = 0;
break;
@ -223,14 +248,38 @@ public class ArbitraryDataManager extends Thread {
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
// Skip transactions that we don't need to proactively store data for
if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) {
ArbitraryDataExamination arbitraryDataExamination = storageManager.shouldPreFetchData(repository, arbitraryTransactionData);
if (!arbitraryDataExamination.isPass()) {
iterator.remove();
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
arbitraryDataExamination.getNotes(),
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
continue;
}
// Remove transactions that we already have local data for
if (hasLocalData(arbitraryTransaction)) {
iterator.remove();
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"already have local data, skipping",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
}
}
@ -248,8 +297,21 @@ public class ArbitraryDataManager extends Thread {
// Check to see if we have had a more recent PUT
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
if (hasMoreRecentPutTransaction) {
Optional<ArbitraryTransactionData> moreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
if (moreRecentPutTransaction.isPresent()) {
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"not fetching old data",
arbitraryTransactionData.getTimestamp(),
moreRecentPutTransaction.get().getTimestamp()
)
);
// There is a more recent PUT transaction than the one we are currently processing.
// When a PUT is issued, it replaces any layers that would have been there before.
// Therefore any data relating to this older transaction is no longer needed and we
@ -257,10 +319,34 @@ public class ArbitraryDataManager extends Thread {
continue;
}
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"fetching data",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
// Ask our connected peers if they have files for this signature
// This process automatically then fetches the files themselves if a peer is found
fetchData(arbitraryTransactionData);
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"fetched data",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
} catch (DataException e) {
LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
}
@ -274,6 +360,20 @@ public class ArbitraryDataManager extends Thread {
final int limit = 100;
int offset = 0;
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
try (final Repository repository = RepositoryManager.getRepository()) {
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactions();
} catch( Exception e) {
LOGGER.error(e.getMessage(), e);
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
}
// collect processed transactions in a set to ensure outdated data transactions do not get fetched
Set<ArbitraryTransactionDataHashWrapper> processedTransactions = new HashSet<>();
while (!isStopping) {
final int minSeconds = 3;
final int maxSeconds = 10;
@ -282,8 +382,8 @@ public class ArbitraryDataManager extends Thread {
// Any arbitrary transactions we want to fetch data for?
try (final Repository repository = RepositoryManager.getRepository()) {
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true);
// LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
List<byte[]> signatures = processTransactionsForSignatures(limit, offset, allArbitraryTransactionsInDescendingOrder, processedTransactions);
if (signatures == null || signatures.isEmpty()) {
offset = 0;
break;
@ -328,26 +428,74 @@ public class ArbitraryDataManager extends Thread {
continue;
}
// Check to see if we have had a more recent PUT
// No longer need to see if we have had a more recent PUT since we compared the transactions to process
// to the transactions previously processed, so we can fetch the transactiondata, notify the event bus,
// fetch the metadata and notify the event bus again
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
if (hasMoreRecentPutTransaction) {
// There is a more recent PUT transaction than the one we are currently processing.
// When a PUT is issued, it replaces any layers that would have been there before.
// Therefore any data relating to this older transaction is no longer needed and we
// shouldn't fetch it from the network.
continue;
}
// Ask our connected peers if they have metadata for this signature
fetchMetadata(arbitraryTransactionData);
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"fetched metadata",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
} catch (DataException e) {
LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
}
private static List<byte[]> processTransactionsForSignatures(
int limit,
int offset,
List<ArbitraryTransactionData> transactionsInDescendingOrder,
Set<ArbitraryTransactionDataHashWrapper> processedTransactions) {
// these transactions are in descending order, latest transactions come first
List<ArbitraryTransactionData> transactions
= transactionsInDescendingOrder.stream()
.skip(offset)
.limit(limit)
.collect(Collectors.toList());
// wrap the transactions, so they can be used for hashing and comparing
// Class ArbitraryTransactionDataHashWrapper supports hashCode() and equals(...) for this purpose
List<ArbitraryTransactionDataHashWrapper> wrappedTransactions
= transactions.stream()
.map(transaction -> new ArbitraryTransactionDataHashWrapper(transaction))
.collect(Collectors.toList());
// create a set of wrappers and populate it first to last, so that all outdated transactions get rejected
Set<ArbitraryTransactionDataHashWrapper> transactionsToProcess = new HashSet<>(wrappedTransactions.size());
for(ArbitraryTransactionDataHashWrapper wrappedTransaction : wrappedTransactions) {
transactionsToProcess.add(wrappedTransaction);
}
// remove the matches for previously processed transactions,
// because these transactions have had updates that have already been processed
transactionsToProcess.removeAll(processedTransactions);
// add to processed transactions to compare and remove matches from future processing iterations
processedTransactions.addAll(transactionsToProcess);
List<byte[]> signatures
= transactionsToProcess.stream()
.map(transactionToProcess -> transactionToProcess.getData()
.getSignature())
.collect(Collectors.toList());
return signatures;
}
private ArbitraryTransaction fetchTransaction(final Repository repository, byte[] signature) {
try {
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);

74
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
View File

@ -155,31 +155,24 @@ public class ArbitraryDataStorageManager extends Thread {
* @param arbitraryTransactionData - the transaction
* @return boolean - whether to prefetch or not
*/
public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
public ArbitraryDataExamination shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
String name = arbitraryTransactionData.getName();
// Only fetch data associated with hashes, as we already have RAW_DATA
if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) {
return false;
return new ArbitraryDataExamination(false, "Only fetch data associated with hashes");
}
// Don't fetch anything more if we're (nearly) out of space
// Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
// avoid a fetch/delete loop
if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) {
return false;
}
// Don't fetch anything if we're (nearly) out of space for this name
// Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
// avoid a fetch/delete loop
if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) {
return false;
return new ArbitraryDataExamination(false,"Don't fetch anything more if we're (nearly) out of space");
}
// Don't store data unless it's an allowed type (public/private)
if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
return false;
return new ArbitraryDataExamination(false, "Don't store data unless it's an allowed type (public/private)");
}
// Handle transactions without names differently
@ -189,21 +182,21 @@ public class ArbitraryDataStorageManager extends Thread {
// Never fetch data from blocked names, even if they are followed
if (ListUtils.isNameBlocked(name)) {
return false;
return new ArbitraryDataExamination(false, "blocked name");
}
switch (Settings.getInstance().getStoragePolicy()) {
case FOLLOWED:
case FOLLOWED_OR_VIEWED:
return ListUtils.isFollowingName(name);
return new ArbitraryDataExamination(ListUtils.isFollowingName(name), Settings.getInstance().getStoragePolicy().name());
case ALL:
return true;
return new ArbitraryDataExamination(true, Settings.getInstance().getStoragePolicy().name());
case NONE:
case VIEWED:
default:
return false;
return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name());
}
}
@ -214,17 +207,17 @@ public class ArbitraryDataStorageManager extends Thread {
*
* @return boolean - whether the storage policy allows for unnamed data
*/
private boolean shouldPreFetchDataWithoutName() {
private ArbitraryDataExamination shouldPreFetchDataWithoutName() {
switch (Settings.getInstance().getStoragePolicy()) {
case ALL:
return true;
return new ArbitraryDataExamination(true, "Fetching all data");
case NONE:
case VIEWED:
case FOLLOWED:
case FOLLOWED_OR_VIEWED:
default:
return false;
return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name());
}
}
@ -484,51 +477,6 @@ public class ArbitraryDataStorageManager extends Thread {
return true;
}
public boolean isStorageSpaceAvailableForName(Repository repository, String name, double threshold) {
if (!this.isStorageSpaceAvailable(threshold)) {
// No storage space available at all, so no need to check this name
return false;
}
if (Settings.getInstance().getStoragePolicy() == StoragePolicy.ALL) {
// Using storage policy ALL, so don't limit anything per name
return true;
}
if (name == null) {
// This transaction doesn't have a name, so fall back to total space limitations
return true;
}
int followedNamesCount = ListUtils.followedNamesCount();
if (followedNamesCount == 0) {
// Not following any names, so we have space
return true;
}
long totalSizeForName = 0;
long maxStoragePerName = this.storageCapacityPerName(threshold);
// Fetch all hosted transactions
List<ArbitraryTransactionData> hostedTransactions = this.listAllHostedTransactions(repository, null, null);
for (ArbitraryTransactionData transactionData : hostedTransactions) {
String transactionName = transactionData.getName();
if (!Objects.equals(name, transactionName)) {
// Transaction relates to a different name
continue;
}
totalSizeForName += transactionData.getSize();
}
// Have we reached the limit for this name?
if (totalSizeForName > maxStoragePerName) {
return false;
}
return true;
}
public long storageCapacityPerName(double threshold) {
int followedNamesCount = ListUtils.followedNamesCount();
if (followedNamesCount == 0) {

48
src/main/java/org/qortal/controller/arbitrary/ArbitraryTransactionDataHashWrapper.java Normal file
View File

@ -0,0 +1,48 @@
package org.qortal.controller.arbitrary;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.transaction.ArbitraryTransactionData;
import java.util.Objects;
public class ArbitraryTransactionDataHashWrapper {
private ArbitraryTransactionData data;
private int service;
private String name;
private String identifier;
public ArbitraryTransactionDataHashWrapper(ArbitraryTransactionData data) {
this.data = data;
this.service = data.getService().value;
this.name = data.getName();
this.identifier = data.getIdentifier();
}
public ArbitraryTransactionDataHashWrapper(int service, String name, String identifier) {
this.service = service;
this.name = name;
this.identifier = identifier;
}
public ArbitraryTransactionData getData() {
return data;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ArbitraryTransactionDataHashWrapper that = (ArbitraryTransactionDataHashWrapper) o;
return service == that.service && name.equals(that.name) && Objects.equals(identifier, that.identifier);
}
@Override
public int hashCode() {
return Objects.hash(service, name, identifier);
}
}

33
src/main/java/org/qortal/controller/arbitrary/RebuildArbitraryResourceCacheTask.java Normal file
View File

@ -0,0 +1,33 @@
package org.qortal.controller.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import java.util.TimerTask;
public class RebuildArbitraryResourceCacheTask extends TimerTask {
private static final Logger LOGGER = LogManager.getLogger(RebuildArbitraryResourceCacheTask.class);
public static final long MILLIS_IN_HOUR = 60 * 60 * 1000;
public static final long MILLIS_IN_MINUTE = 60 * 1000;
private static final String REBUILD_ARBITRARY_RESOURCE_CACHE_TASK = "Rebuild Arbitrary Resource Cache Task";
@Override
public void run() {
Thread.currentThread().setName(REBUILD_ARBITRARY_RESOURCE_CACHE_TASK);
try (final Repository repository = RepositoryManager.getRepository()) {
ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, true);
}
catch( DataException e ) {
LOGGER.error(e.getMessage(), e);
}
}
}

1
src/main/java/org/qortal/crosschain/Bitcoiny.java
View File

@ -83,6 +83,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
return this.bitcoinjContext;
}
@Override
public String getCurrencyCode() {
return this.currencyCode;
}

2
src/main/java/org/qortal/crosschain/ForeignBlockchain.java
View File

@ -2,6 +2,8 @@ package org.qortal.crosschain;
public interface ForeignBlockchain {
public String getCurrencyCode();
public boolean isValidAddress(String address);
public boolean isValidWalletKey(String walletKey);

34
src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java Normal file
View File

@ -0,0 +1,34 @@
package org.qortal.data.arbitrary;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndex {
public String t;
public String n;
public int c;
public String l;
public ArbitraryDataIndex() {}
public ArbitraryDataIndex(String t, String n, int c, String l) {
this.t = t;
this.n = n;
this.c = c;
this.l = l;
}
@Override
public String toString() {
return "ArbitraryDataIndex{" +
"t='" + t + '\'' +
", n='" + n + '\'' +
", c=" + c +
", l='" + l + '\'' +
'}';
}
}

41
src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java Normal file
View File

@ -0,0 +1,41 @@
package org.qortal.data.arbitrary;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndexDetail {
public String issuer;
public int rank;
public String term;
public String name;
public int category;
public String link;
public String indexIdentifer;
public ArbitraryDataIndexDetail() {}
public ArbitraryDataIndexDetail(String issuer, int rank, ArbitraryDataIndex index, String indexIdentifer) {
this.issuer = issuer;
this.rank = rank;
this.term = index.t;
this.name = index.n;
this.category = index.c;
this.link = index.l;
this.indexIdentifer = indexIdentifer;
}
@Override
public String toString() {
return "ArbitraryDataIndexDetail{" +
"issuer='" + issuer + '\'' +
", rank=" + rank +
", term='" + term + '\'' +
", name='" + name + '\'' +
", category=" + category +
", link='" + link + '\'' +
", indexIdentifer='" + indexIdentifer + '\'' +
'}';
}
}

38
src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java Normal file
View File

@ -0,0 +1,38 @@
package org.qortal.data.arbitrary;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.Objects;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndexScoreKey {
public String name;
public int category;
public String link;
public ArbitraryDataIndexScoreKey() {}
public ArbitraryDataIndexScoreKey(String name, int category, String link) {
this.name = name;
this.category = category;
this.link = link;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ArbitraryDataIndexScoreKey that = (ArbitraryDataIndexScoreKey) o;
return category == that.category && Objects.equals(name, that.name) && Objects.equals(link, that.link);
}
@Override
public int hashCode() {
return Objects.hash(name, category, link);
}
}

38
src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java Normal file
View File

@ -0,0 +1,38 @@
package org.qortal.data.arbitrary;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndexScorecard {
public double score;
public String name;
public int category;
public String link;
public ArbitraryDataIndexScorecard() {}
public ArbitraryDataIndexScorecard(double score, String name, int category, String link) {
this.score = score;
this.name = name;
this.category = category;
this.link = link;
}
public double getScore() {
return score;
}
@Override
public String toString() {
return "ArbitraryDataIndexScorecard{" +
"score=" + score +
", name='" + name + '\'' +
", category=" + category +
", link='" + link + '\'' +
'}';
}
}

57
src/main/java/org/qortal/data/arbitrary/DataMonitorInfo.java Normal file
View File

@ -0,0 +1,57 @@
package org.qortal.data.arbitrary;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class DataMonitorInfo {
private long timestamp;
private String identifier;
private String name;
private String service;
private String description;
private long transactionTimestamp;
private long latestPutTimestamp;
public DataMonitorInfo() {
}
public DataMonitorInfo(long timestamp, String identifier, String name, String service, String description, long transactionTimestamp, long latestPutTimestamp) {
this.timestamp = timestamp;
this.identifier = identifier;
this.name = name;
this.service = service;
this.description = description;
this.transactionTimestamp = transactionTimestamp;
this.latestPutTimestamp = latestPutTimestamp;
}
public long getTimestamp() {
return timestamp;
}
public String getIdentifier() {
return identifier;
}
public String getName() {
return name;
}
public String getService() {
return service;
}
public String getDescription() {
return description;
}
public long getTransactionTimestamp() {
return transactionTimestamp;
}
public long getLatestPutTimestamp() {
return latestPutTimestamp;
}
}

23
src/main/java/org/qortal/data/arbitrary/IndexCache.java Normal file
View File

@ -0,0 +1,23 @@
package org.qortal.data.arbitrary;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
public class IndexCache {
public static final IndexCache SINGLETON = new IndexCache();
private ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> indicesByTerm = new ConcurrentHashMap<>();
private ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> indicesByIssuer = new ConcurrentHashMap<>();
public static IndexCache getInstance() {
return SINGLETON;
}
public ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> getIndicesByTerm() {
return indicesByTerm;
}
public ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> getIndicesByIssuer() {
return indicesByIssuer;
}
}

22
src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java
View File

@ -200,4 +200,26 @@ public class ArbitraryTransactionData extends TransactionData {
return this.payments;
}
@Override
public String toString() {
return "ArbitraryTransactionData{" +
"version=" + version +
", service=" + service +
", nonce=" + nonce +
", size=" + size +
", name='" + name + '\'' +
", identifier='" + identifier + '\'' +
", method=" + method +
", compression=" + compression +
", dataType=" + dataType +
", type=" + type +
", timestamp=" + timestamp +
", fee=" + fee +
", txGroupId=" + txGroupId +
", blockHeight=" + blockHeight +
", blockSequence=" + blockSequence +
", approvalStatus=" + approvalStatus +
", approvalHeight=" + approvalHeight +
'}';
}
}

57
src/main/java/org/qortal/event/DataMonitorEvent.java Normal file
View File

@ -0,0 +1,57 @@
package org.qortal.event;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class DataMonitorEvent implements Event{
private long timestamp;
private String identifier;
private String name;
private String service;
private String description;
private long transactionTimestamp;
private long latestPutTimestamp;
public DataMonitorEvent() {
}
public DataMonitorEvent(long timestamp, String identifier, String name, String service, String description, long transactionTimestamp, long latestPutTimestamp) {
this.timestamp = timestamp;
this.identifier = identifier;
this.name = name;
this.service = service;
this.description = description;
this.transactionTimestamp = transactionTimestamp;
this.latestPutTimestamp = latestPutTimestamp;
}
public long getTimestamp() {
return timestamp;
}
public String getIdentifier() {
return identifier;
}
public String getName() {
return name;
}
public String getService() {
return service;
}
public String getDescription() {
return description;
}
public long getTransactionTimestamp() {
return transactionTimestamp;
}
public long getLatestPutTimestamp() {
return latestPutTimestamp;
}
}

4
src/main/java/org/qortal/group/Group.java
View File

@ -674,8 +674,8 @@ public class Group {
public void uninvite(GroupInviteTransactionData groupInviteTransactionData) throws DataException {
String invitee = groupInviteTransactionData.getInvitee();
// If member exists then they were added when invite matched join request
if (this.memberExists(invitee)) {
// If member exists and the join request is present then they were added when invite matched join request
if (this.memberExists(invitee) && groupInviteTransactionData.getJoinReference() != null) {
// Rebuild join request using cached reference to transaction that created join request.
this.rebuildJoinRequest(invitee, groupInviteTransactionData.getJoinReference());

6
src/main/java/org/qortal/repository/ATRepository.java
View File

@ -76,9 +76,9 @@ public interface ATRepository {
* Although <tt>expectedValue</tt>, if provided, is natively an unsigned long,
* the data segment comparison is done via unsigned hex string.
*/
public List<ATStateData> getMatchingFinalATStates(byte[] codeHash, String buyerAddress, String sellerAddress, Boolean isFinished,
Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight,
Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ATStateData> getMatchingFinalATStates(byte[] codeHash, byte[] buyerPublicKey, byte[] sellerPublicKey, Boolean isFinished,
Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight,
Integer limit, Integer offset, Boolean reverse) throws DataException;
/**
* Returns final ATStateData for ATs matching codeHash (required)

6
src/main/java/org/qortal/repository/ArbitraryRepository.java
View File

@ -27,6 +27,10 @@ public interface ArbitraryRepository {
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
List<ArbitraryTransactionData> getLatestArbitraryTransactions() throws DataException;
List<ArbitraryTransactionData> getLatestArbitraryTransactionsByName(String name) throws DataException;
public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException;
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException;
@ -42,7 +46,7 @@ public interface ArbitraryRepository {
public List<ArbitraryResourceData> getArbitraryResources(Service service, String identifier, List<String> names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, List<String> keywords, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException;
List<ArbitraryResourceData> searchArbitraryResourcesSimple(
Service service,

29
src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java
View File

@ -5,6 +5,7 @@ import com.google.common.primitives.Longs;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.crypto.Crypto;
import org.qortal.data.at.ATData;
import org.qortal.data.at.ATStateData;
import org.qortal.repository.ATRepository;
@ -403,9 +404,9 @@ public class HSQLDBATRepository implements ATRepository {
}
@Override
public List<ATStateData> getMatchingFinalATStates(byte[] codeHash, String buyerAddress, String sellerAddress, Boolean isFinished,
Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight,
Integer limit, Integer offset, Boolean reverse) throws DataException {
public List<ATStateData> getMatchingFinalATStates(byte[] codeHash, byte[] buyerPublicKey, byte[] sellerPublicKey, Boolean isFinished,
Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight,
Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(1024);
List<Object> bindParams = new ArrayList<>();
@ -426,9 +427,9 @@ public class HSQLDBATRepository implements ATRepository {
// Both must be the same direction (DESC) also
sql.append("ORDER BY ATStates.height DESC LIMIT 1) AS FinalATStates ");
// Optional LEFT JOIN with ATTRANSACTIONS for buyerAddress
if (buyerAddress != null && !buyerAddress.isEmpty()) {
sql.append("LEFT JOIN ATTRANSACTIONS tx ON tx.at_address = ATs.AT_address ");
// Optional JOIN with ATTRANSACTIONS for buyerAddress
if (buyerPublicKey != null && buyerPublicKey.length > 0) {
sql.append("JOIN ATTRANSACTIONS tx ON tx.at_address = ATs.AT_address ");
}
sql.append("WHERE ATs.code_hash = ? ");
@ -450,18 +451,18 @@ public class HSQLDBATRepository implements ATRepository {
bindParams.add(rawExpectedValue);
}
if (buyerAddress != null && !buyerAddress.isEmpty()) {
sql.append("AND tx.recipient = ? ");
bindParams.add(buyerAddress);
if (buyerPublicKey != null && buyerPublicKey.length > 0 ) {
// the buyer must be the recipient of the transaction and not the creator of the AT
sql.append("AND tx.recipient = ? AND ATs.creator != ? ");
bindParams.add(Crypto.toAddress(buyerPublicKey));
bindParams.add(buyerPublicKey);
}
if (sellerAddress != null && !sellerAddress.isEmpty()) {
// Convert sellerAddress to publicKey (method depends on your implementation)
AccountData accountData = this.repository.getAccountRepository().getAccount(sellerAddress);
byte[] publicKey = accountData.getPublicKey();
if (sellerPublicKey != null && sellerPublicKey.length > 0) {
sql.append("AND ATs.creator = ? ");
bindParams.add(publicKey);
bindParams.add(sellerPublicKey);
}
sql.append(" ORDER BY FinalATStates.height ");

165
src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java
View File

@ -7,7 +7,6 @@ import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Category;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.data.arbitrary.ArbitraryResourceCache;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
@ -29,6 +28,7 @@ import org.qortal.utils.ListUtils;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@ -227,6 +227,144 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
}
@Override
public List<ArbitraryTransactionData> getLatestArbitraryTransactions() throws DataException {
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
"tx_group_id, block_height, approval_status, approval_height, " +
"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " +
"WHERE name IS NOT NULL " +
"ORDER BY created_when DESC";
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql)) {
if (resultSet == null)
return new ArrayList<>(0);
do {
byte[] reference = resultSet.getBytes(2);
byte[] signature = resultSet.getBytes(3);
byte[] creatorPublicKey = resultSet.getBytes(4);
long timestamp = resultSet.getLong(5);
Long fee = resultSet.getLong(6);
if (fee == 0 && resultSet.wasNull())
fee = null;
int txGroupId = resultSet.getInt(7);
Integer blockHeight = resultSet.getInt(8);
if (blockHeight == 0 && resultSet.wasNull())
blockHeight = null;
ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
Integer approvalHeight = resultSet.getInt(10);
if (approvalHeight == 0 && resultSet.wasNull())
approvalHeight = null;
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
int version = resultSet.getInt(11);
int nonce = resultSet.getInt(12);
int serviceInt = resultSet.getInt(13);
int size = resultSet.getInt(14);
boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
byte[] data = resultSet.getBytes(16);
byte[] metadataHash = resultSet.getBytes(17);
String nameResult = resultSet.getString(18);
String identifierResult = resultSet.getString(19);
Method method = Method.valueOf(resultSet.getInt(20));
byte[] secret = resultSet.getBytes(21);
Compression compression = Compression.valueOf(resultSet.getInt(22));
// FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls.
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
version, serviceInt, nonce, size, nameResult, identifierResult, method, secret,
compression, data, dataType, metadataHash, null);
arbitraryTransactionData.add(transactionData);
} while (resultSet.next());
return arbitraryTransactionData;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
return new ArrayList<>(0);
}
}
@Override
public List<ArbitraryTransactionData> getLatestArbitraryTransactionsByName( String name ) throws DataException {
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
"tx_group_id, block_height, approval_status, approval_height, " +
"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " +
"WHERE name = ? " +
"ORDER BY created_when DESC";
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql, name)) {
if (resultSet == null)
return new ArrayList<>(0);
do {
byte[] reference = resultSet.getBytes(2);
byte[] signature = resultSet.getBytes(3);
byte[] creatorPublicKey = resultSet.getBytes(4);
long timestamp = resultSet.getLong(5);
Long fee = resultSet.getLong(6);
if (fee == 0 && resultSet.wasNull())
fee = null;
int txGroupId = resultSet.getInt(7);
Integer blockHeight = resultSet.getInt(8);
if (blockHeight == 0 && resultSet.wasNull())
blockHeight = null;
ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
Integer approvalHeight = resultSet.getInt(10);
if (approvalHeight == 0 && resultSet.wasNull())
approvalHeight = null;
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
int version = resultSet.getInt(11);
int nonce = resultSet.getInt(12);
int serviceInt = resultSet.getInt(13);
int size = resultSet.getInt(14);
boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
byte[] data = resultSet.getBytes(16);
byte[] metadataHash = resultSet.getBytes(17);
String nameResult = resultSet.getString(18);
String identifierResult = resultSet.getString(19);
Method method = Method.valueOf(resultSet.getInt(20));
byte[] secret = resultSet.getBytes(21);
Compression compression = Compression.valueOf(resultSet.getInt(22));
// FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls.
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
version, serviceInt, nonce, size, nameResult, identifierResult, method, secret,
compression, data, dataType, metadataHash, null);
arbitraryTransactionData.add(transactionData);
} while (resultSet.next());
return arbitraryTransactionData;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
return new ArrayList<>(0);
}
}
private ArbitraryTransactionData getSingleTransaction(String name, Service service, Method method, String identifier, boolean firstNotLast) throws DataException {
if (name == null || service == null) {
// Required fields
@ -724,12 +862,11 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
@Override
public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, boolean prefixOnly,
public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, List<String> keywords, boolean prefixOnly,
List<String> exactMatchNames, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked,
Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException {
if(Settings.getInstance().isDbCacheEnabled()) {
List<ArbitraryResourceData> list
= HSQLDBCacheUtils.callCache(
ArbitraryResourceCache.getInstance(),
@ -751,6 +888,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
Optional.ofNullable(description),
prefixOnly,
Optional.ofNullable(exactMatchNames),
Optional.ofNullable(keywords),
defaultResource,
Optional.ofNullable(minLevel),
Optional.ofNullable(() -> ListUtils.followedNames()),
@ -771,6 +909,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
}
StringBuilder sql = new StringBuilder(512);
List<Object> bindParams = new ArrayList<>();
@ -857,6 +996,26 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
bindParams.add(queryWildcard);
}
if (keywords != null && !keywords.isEmpty()) {
List<String> searchKeywords = new ArrayList<>(keywords);
List<String> conditions = new ArrayList<>();
List<String> bindValues = new ArrayList<>();
for (int i = 0; i < searchKeywords.size(); i++) {
conditions.add("LOWER(description) LIKE ?");
bindValues.add("%" + searchKeywords.get(i).trim().toLowerCase() + "%");
}
String finalCondition = String.join(" OR ", conditions);
sql.append(" AND (").append(finalCondition).append(")");
bindParams.addAll(bindValues);
}
// Handle name searches
if (names != null && !names.isEmpty()) {
sql.append(" AND (");

96
src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java
View File

@ -167,6 +167,7 @@ public class HSQLDBCacheUtils {
Optional<String> description,
boolean prefixOnly,
Optional<List<String>> exactMatchNames,
Optional<List<String>> keywords,
boolean defaultResource,
Optional<Integer> minLevel,
Optional<Supplier<List<String>>> includeOnly,
@ -182,6 +183,14 @@ public class HSQLDBCacheUtils {
// retain only candidates with names
Stream<ArbitraryResourceData> stream = candidates.stream().filter(candidate -> candidate.name != null );
if(after.isPresent()) {
stream = stream.filter( candidate -> candidate.created > after.get().longValue() );
}
if(before.isPresent()) {
stream = stream.filter( candidate -> candidate.created < before.get().longValue() );
}
if(exclude.isPresent())
stream = stream.filter( candidate -> !exclude.get().get().contains( candidate.name ));
@ -207,6 +216,36 @@ public class HSQLDBCacheUtils {
stream = filterTerm(title, data -> data.metadata != null ? data.metadata.getTitle() : null, prefixOnly, stream);
stream = filterTerm(description, data -> data.metadata != null ? data.metadata.getDescription() : null, prefixOnly, stream);
// New: Filter by keywords if provided
if (keywords.isPresent() && !keywords.get().isEmpty()) {
List<String> searchKeywords = keywords.get().stream()
.map(String::toLowerCase)
.collect(Collectors.toList());
stream = stream.filter(candidate -> {
if (candidate.metadata != null && candidate.metadata.getDescription() != null) {
String descriptionLower = candidate.metadata.getDescription().toLowerCase();
return searchKeywords.stream().anyMatch(descriptionLower::contains);
}
return false;
});
}
if (keywords.isPresent() && !keywords.get().isEmpty()) {
List<String> searchKeywords = keywords.get().stream()
.map(String::toLowerCase)
.collect(Collectors.toList());
stream = stream.filter(candidate -> {
if (candidate.metadata != null && candidate.metadata.getDescription() != null) {
String descriptionLower = candidate.metadata.getDescription().toLowerCase();
return searchKeywords.stream().anyMatch(descriptionLower::contains);
}
return false;
});
}
// if exact names is set, retain resources with exact names
if( exactMatchNames.isPresent() && !exactMatchNames.get().isEmpty()) {
@ -262,15 +301,58 @@ public class HSQLDBCacheUtils {
// truncate to limit
if( limit.isPresent() && limit.get() > 0 ) stream = stream.limit(limit.get());
// include metadata
if( includeMetadata.isEmpty() || !includeMetadata.get() )
stream = stream.peek( candidate -> candidate.metadata = null );
List<ArbitraryResourceData> listCopy1 = stream.collect(Collectors.toList());
// include status
if( includeStatus.isEmpty() || !includeStatus.get() )
stream = stream.peek( candidate -> candidate.status = null);
List<ArbitraryResourceData> listCopy2 = new ArrayList<>(listCopy1.size());
return stream.collect(Collectors.toList());
// remove metadata from the first copy
if( includeMetadata.isEmpty() || !includeMetadata.get() ) {
for( ArbitraryResourceData data : listCopy1 ) {
ArbitraryResourceData copy = new ArbitraryResourceData();
copy.name = data.name;
copy.service = data.service;
copy.identifier = data.identifier;
copy.status = data.status;
copy.metadata = null;
copy.size = data.size;
copy.created = data.created;
copy.updated = data.updated;
listCopy2.add(copy);
}
}
// put the list copy 1 into the second copy
else {
listCopy2.addAll(listCopy1);
}
// remove status from final copy
if( includeStatus.isEmpty() || !includeStatus.get() ) {
List<ArbitraryResourceData> finalCopy = new ArrayList<>(listCopy2.size());
for( ArbitraryResourceData data : listCopy2 ) {
ArbitraryResourceData copy = new ArbitraryResourceData();
copy.name = data.name;
copy.service = data.service;
copy.identifier = data.identifier;
copy.status = null;
copy.metadata = data.metadata;
copy.size = data.size;
copy.created = data.created;
copy.updated = data.updated;
finalCopy.add(copy);
}
return finalCopy;
}
// keep status included by returning the second copy
else {
return listCopy2;
}
}
/**

65
src/main/java/org/qortal/settings/Settings.java
View File

@ -386,7 +386,7 @@ public class Settings {
/**
* DB Cache Enabled?
*/
private boolean dbCacheEnabled = false;
private boolean dbCacheEnabled = true;
/**
* DB Cache Thread Priority
@ -508,7 +508,44 @@ public class Settings {
*/
private boolean connectionPoolMonitorEnabled = false;
// Domain mapping
/**
* Buiild Arbitrary Resources Batch Size
*
* The number resources to batch per iteration when rebuilding.
*/
private int buildArbitraryResourcesBatchSize = 200;
/**
* Arbitrary Indexing Priority
*
* The thread priority when indexing arbirary resources.
*/
private int arbitraryIndexingPriority = 5;
/**
* Arbitrary Indexing Frequency (In Minutes)
*
* The frequency at which the arbitrary indices are cached.
*/
private int arbitraryIndexingFrequency = 10;
private boolean rebuildArbitraryResourceCacheTaskEnabled = false;
/**
* Rebuild Arbitrary Resource Cache Task Delay (In Minutes)
*
* Waiting period before the first rebuild task is started.
*/
private int rebuildArbitraryResourceCacheTaskDelay = 300;
/**
* Rebuild Arbitrary Resource Cache Task Period (In Hours)
*
* The frequency the arbitrary resource cache is rebuilt.
*/
private int rebuildArbitraryResourceCacheTaskPeriod = 24;
// Domain mapping
public static class ThreadLimit {
private String messageType;
private Integer limit;
@ -1333,4 +1370,28 @@ public class Settings {
public boolean isConnectionPoolMonitorEnabled() {
return connectionPoolMonitorEnabled;
}
public int getBuildArbitraryResourcesBatchSize() {
return buildArbitraryResourcesBatchSize;
}
public int getArbitraryIndexingPriority() {
return arbitraryIndexingPriority;
}
public int getArbitraryIndexingFrequency() {
return arbitraryIndexingFrequency;
}
public boolean isRebuildArbitraryResourceCacheTaskEnabled() {
return rebuildArbitraryResourceCacheTaskEnabled;
}
public int getRebuildArbitraryResourceCacheTaskDelay() {
return rebuildArbitraryResourceCacheTaskDelay;
}
public int getRebuildArbitraryResourceCacheTaskPeriod() {
return rebuildArbitraryResourceCacheTaskPeriod;
}
}

148
src/main/java/org/qortal/transaction/ArbitraryTransaction.java
View File

@ -9,6 +9,7 @@ import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Service;
import org.qortal.block.BlockChain;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.controller.arbitrary.ArbitraryTransactionDataHashWrapper;
import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
import org.qortal.crypto.Crypto;
import org.qortal.crypto.MemoryPoW;
@ -31,8 +32,12 @@ import org.qortal.utils.ArbitraryTransactionUtils;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
public class ArbitraryTransaction extends Transaction {
@ -303,8 +308,13 @@ public class ArbitraryTransaction extends Transaction {
// Add/update arbitrary resource caches, but don't update the status as this involves time-consuming
// disk reads, and is more prone to failure. The status will be updated on metadata retrieval, or when
// accessing the resource.
this.updateArbitraryResourceCache(repository);
this.updateArbitraryMetadataCache(repository);
// Also, must add this transaction as a latest transaction, since the it has not been saved to the
// repository yet.
this.updateArbitraryResourceCacheIncludingMetadata(
repository,
Set.of(new ArbitraryTransactionDataHashWrapper(arbitraryTransactionData)),
new HashMap<>(0)
);
repository.saveChanges();
@ -360,7 +370,10 @@ public class ArbitraryTransaction extends Transaction {
*
* @throws DataException
*/
public void updateArbitraryResourceCache(Repository repository) throws DataException {
public void updateArbitraryResourceCacheIncludingMetadata(
Repository repository,
Set<ArbitraryTransactionDataHashWrapper> latestTransactionWrappers,
Map<ArbitraryTransactionDataHashWrapper, ArbitraryResourceData> resourceByWrapper) throws DataException {
// Don't cache resources without a name (such as auto updates)
if (arbitraryTransactionData.getName() == null) {
return;
@ -385,29 +398,42 @@ public class ArbitraryTransaction extends Transaction {
arbitraryResourceData.name = name;
arbitraryResourceData.identifier = identifier;
// Get the latest transaction
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) {
LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData);
// We don't have a latest transaction, so delete from cache
repository.getArbitraryRepository().delete(arbitraryResourceData);
return;
final ArbitraryTransactionDataHashWrapper wrapper = new ArbitraryTransactionDataHashWrapper(arbitraryTransactionData);
ArbitraryTransactionData latestTransactionData;
if( latestTransactionWrappers.contains(wrapper)) {
latestTransactionData
= latestTransactionWrappers.stream()
.filter( latestWrapper -> latestWrapper.equals(wrapper))
.findAny().get()
.getData();
}
else {
// Get the latest transaction
latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) {
LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData);
// We don't have a latest transaction, so delete from cache
repository.getArbitraryRepository().delete(arbitraryResourceData);
return;
}
}
ArbitraryResourceData existingArbitraryResourceData = resourceByWrapper.get(wrapper);
// Get existing cached entry if it exists
ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository()
.getArbitraryResource(service, name, identifier);
LOGGER.info("updating existing arbitraryResourceData" + existingArbitraryResourceData);
if( existingArbitraryResourceData == null ) {
// Get existing cached entry if it exists
existingArbitraryResourceData = repository.getArbitraryRepository()
.getArbitraryResource(service, name, identifier);
}
// Check for existing cached data
if (existingArbitraryResourceData == null) {
// Nothing exists yet, so set creation date from the current transaction (it will be reduced later if needed)
arbitraryResourceData.created = arbitraryTransactionData.getTimestamp();
arbitraryResourceData.updated = null;
LOGGER.info("updated = null, reason = existingArbitraryResourceData == null" );
}
else {
resourceByWrapper.put(wrapper, existingArbitraryResourceData);
// An entry already exists - update created time from current transaction if this is older
arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp());
@ -415,22 +441,44 @@ public class ArbitraryTransaction extends Transaction {
if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) {
// Latest transaction matches created time, so it hasn't been updated
arbitraryResourceData.updated = null;
LOGGER.info(
"updated = null, reason: existingArbitraryResourceData.created == latestTransactionData.getTimestamp() == " +
existingArbitraryResourceData.created );
}
else {
arbitraryResourceData.updated = latestTransactionData.getTimestamp();
LOGGER.info("setting updated to a non-null value");
}
}
arbitraryResourceData.size = latestTransactionData.getSize();
LOGGER.info("saving updated arbitraryResourceData: updated = " + arbitraryResourceData.updated);
// Save
repository.getArbitraryRepository().save(arbitraryResourceData);
// Update metadata for latest transaction if it is local
if (latestTransactionData.getMetadataHash() != null) {
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
if (metadataFile.exists()) {
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
try {
transactionMetadata.read();
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
metadata.setTitle(transactionMetadata.getTitle());
metadata.setDescription(transactionMetadata.getDescription());
metadata.setCategory(transactionMetadata.getCategory());
metadata.setTags(transactionMetadata.getTags());
repository.getArbitraryRepository().save(metadata);
} catch (IOException e) {
// Ignore, as we can add it again later
}
} else {
// We don't have a local copy of this metadata file, so delete it from the cache
// It will be re-added if the file later arrives via the network
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
repository.getArbitraryRepository().delete(metadata);
}
}
}
public void updateArbitraryResourceStatus(Repository repository) throws DataException {
@ -465,60 +513,4 @@ public class ArbitraryTransaction extends Transaction {
repository.getArbitraryRepository().setStatus(arbitraryResourceData, status);
}
public void updateArbitraryMetadataCache(Repository repository) throws DataException {
// Get the latest transaction
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) {
// We don't have a latest transaction, so give up
return;
}
Service service = latestTransactionData.getService();
String name = latestTransactionData.getName();
String identifier = latestTransactionData.getIdentifier();
if (service == null) {
// Unsupported service - ignore this resource
return;
}
// In the cache we store null identifiers as "default", as it is part of the primary key
if (identifier == null) {
identifier = "default";
}
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.service = service;
arbitraryResourceData.name = name;
arbitraryResourceData.identifier = identifier;
// Update metadata for latest transaction if it is local
if (latestTransactionData.getMetadataHash() != null) {
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
if (metadataFile.exists()) {
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
try {
transactionMetadata.read();
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
metadata.setTitle(transactionMetadata.getTitle());
metadata.setDescription(transactionMetadata.getDescription());
metadata.setCategory(transactionMetadata.getCategory());
metadata.setTags(transactionMetadata.getTags());
repository.getArbitraryRepository().save(metadata);
} catch (IOException e) {
// Ignore, as we can add it again later
}
} else {
// We don't have a local copy of this metadata file, so delete it from the cache
// It will be re-added if the file later arrives via the network
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
repository.getArbitraryRepository().delete(metadata);
}
}
}
}

250
src/main/java/org/qortal/utils/ArbitraryIndexUtils.java Normal file
View File

@ -0,0 +1,250 @@
package org.qortal.utils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.exc.InvalidFormatException;
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.SearchMode;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryDataIndex;
import org.qortal.data.arbitrary.ArbitraryDataIndexDetail;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.IndexCache;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class ArbitraryIndexUtils {
public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final Logger LOGGER = LogManager.getLogger(ArbitraryIndexUtils.class);
public static final String INDEX_CACHE_TIMER = "Arbitrary Index Cache Timer";
public static final String INDEX_CACHE_TIMER_TASK = "Arbitrary Index Cache Timer Task";
public static void startCaching(int priorityRequested, int frequency) {
Timer timer = buildTimer(INDEX_CACHE_TIMER, priorityRequested);
TimerTask task = new TimerTask() {
@Override
public void run() {
Thread.currentThread().setName(INDEX_CACHE_TIMER_TASK);
try {
fillCache(IndexCache.getInstance());
} catch (IOException | DataException e) {
LOGGER.error(e.getMessage(), e);
}
}
};
// delay 1 second
timer.scheduleAtFixedRate(task, 1_000, frequency * 60_000);
}
private static void fillCache(IndexCache instance) throws DataException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
List<ArbitraryResourceData> indexResources
= repository.getArbitraryRepository().searchArbitraryResources(
Service.JSON,
null,
"idx-",
null,
null,
null,
null,
true,
null,
false,
SearchMode.ALL,
0,
null,
null,
null,
null,
null,
null,
null,
null,
true);
List<ArbitraryDataIndexDetail> indexDetails = new ArrayList<>();
LOGGER.debug("processing index resource data: count = " + indexResources.size());
// process all index resources
for( ArbitraryResourceData indexResource : indexResources ) {
try {
LOGGER.debug("processing index resource: name = " + indexResource.name + ", identifier = " + indexResource.identifier);
String json = ArbitraryIndexUtils.getJson(indexResource.name, indexResource.identifier);
// map the JSON string to a list of Java objects
List<ArbitraryDataIndex> indices = OBJECT_MAPPER.readValue(json, new TypeReference<List<ArbitraryDataIndex>>() {});
LOGGER.debug("processed indices = " + indices);
// rank and create index detail for each index in this index resource
for( int rank = 1; rank <= indices.size(); rank++ ) {
indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1), indexResource.identifier ));
}
} catch (InvalidFormatException e) {
LOGGER.debug("invalid format, skipping: " + indexResource);
} catch (UnrecognizedPropertyException e) {
LOGGER.debug("unrecognized property, skipping " + indexResource);
}
}
LOGGER.debug("processing indices by term ...");
Map<String, List<ArbitraryDataIndexDetail>> indicesByTerm
= indexDetails.stream().collect(
Collectors.toMap(
detail -> detail.term, // map by term
detail -> List.of(detail), // create list for term
(list1, list2) // merge lists for same term
-> Stream.of(list1, list2)
.flatMap(List::stream)
.collect(Collectors.toList())
)
);
LOGGER.info("processed indices by term: count = " + indicesByTerm.size());
// lock, clear old, load new
synchronized( IndexCache.getInstance().getIndicesByTerm() ) {
IndexCache.getInstance().getIndicesByTerm().clear();
IndexCache.getInstance().getIndicesByTerm().putAll(indicesByTerm);
}
LOGGER.info("loaded indices by term");
LOGGER.debug("processing indices by issuer ...");
Map<String, List<ArbitraryDataIndexDetail>> indicesByIssuer
= indexDetails.stream().collect(
Collectors.toMap(
detail -> detail.issuer, // map by issuer
detail -> List.of(detail), // create list for issuer
(list1, list2) // merge lists for same issuer
-> Stream.of(list1, list2)
.flatMap(List::stream)
.collect(Collectors.toList())
)
);
LOGGER.info("processed indices by issuer: count = " + indicesByIssuer.size());
// lock, clear old, load new
synchronized( IndexCache.getInstance().getIndicesByIssuer() ) {
IndexCache.getInstance().getIndicesByIssuer().clear();
IndexCache.getInstance().getIndicesByIssuer().putAll(indicesByIssuer);
}
LOGGER.info("loaded indices by issuer");
}
}
private static Timer buildTimer( final String name, int priorityRequested) {
// ensure priority is in between 1-10
final int priority = Math.max(0, Math.min(10, priorityRequested));
// Create a custom Timer with updated priority threads
Timer timer = new Timer(true) { // 'true' to make the Timer daemon
@Override
public void schedule(TimerTask task, long delay) {
Thread thread = new Thread(task, name) {
@Override
public void run() {
this.setPriority(priority);
super.run();
}
};
thread.setPriority(priority);
thread.start();
}
};
return timer;
}
public static String getJsonWithExceptionHandling( String name, String identifier ) {
try {
return getJson(name, identifier);
}
catch( Exception e ) {
LOGGER.error(e.getMessage(), e);
return e.getMessage();
}
}
public static String getJson(String name, String identifier) throws IOException {
try {
ArbitraryDataReader arbitraryDataReader
= new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, Service.JSON, identifier);
int attempts = 0;
Integer maxAttempts = 5;
while (!Controller.isStopping()) {
attempts++;
if (!arbitraryDataReader.isBuilding()) {
try {
arbitraryDataReader.loadSynchronously(false);
break;
} catch (MissingDataException e) {
if (attempts > maxAttempts) {
// Give up after 5 attempts
throw new IOException("Data unavailable. Please try again later.");
}
}
}
Thread.sleep(3000L);
}
java.nio.file.Path outputPath = arbitraryDataReader.getFilePath();
if (outputPath == null) {
// Assume the resource doesn't exist
throw new IOException( "File not found");
}
// No file path supplied - so check if this is a single file resource
String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal");
String filepath = files[0];
java.nio.file.Path path = Paths.get(outputPath.toString(), filepath);
if (!Files.exists(path)) {
String message = String.format("No file exists at filepath: %s", filepath);
throw new IOException( message );
}
String data = Files.readString(path);
return data;
} catch (Exception e) {
throw new IOException(String.format("Unable to load %s %s: %s", Service.JSON, name, e.getMessage()));
}
}
}

24
src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java
View File

@ -24,6 +24,7 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
@ -72,23 +73,23 @@ public class ArbitraryTransactionUtils {
return latestPut;
}
public static boolean hasMoreRecentPutTransaction(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
public static Optional<ArbitraryTransactionData> hasMoreRecentPutTransaction(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
byte[] signature = arbitraryTransactionData.getSignature();
if (signature == null) {
// We can't make a sensible decision without a signature
// so it's best to assume there is nothing newer
return false;
return Optional.empty();
}
ArbitraryTransactionData latestPut = ArbitraryTransactionUtils.fetchLatestPut(repository, arbitraryTransactionData);
if (latestPut == null) {
return false;
return Optional.empty();
}
// If the latest PUT transaction has a newer timestamp, it will override the existing transaction
// Any data relating to the older transaction is no longer needed
boolean hasNewerPut = (latestPut.getTimestamp() > arbitraryTransactionData.getTimestamp());
return hasNewerPut;
return hasNewerPut ? Optional.of(latestPut) : Optional.empty();
}
public static boolean completeFileExists(ArbitraryTransactionData transactionData) throws DataException {
@ -208,7 +209,15 @@ public class ArbitraryTransactionUtils {
return ArbitraryTransactionUtils.isFileRecent(filePath, now, cleanupAfter);
}
public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
/**
*
* @param arbitraryTransactionData
* @param now
* @param cleanupAfter
* @return true if file is deleted, otherwise return false
* @throws DataException
*/
public static boolean deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
byte[] completeHash = arbitraryTransactionData.getData();
byte[] signature = arbitraryTransactionData.getSignature();
@ -219,6 +228,11 @@ public class ArbitraryTransactionUtils {
"if needed", Base58.encode(completeHash));
arbitraryDataFile.delete();
return true;
}
else {
return false;
}
}

8
src/main/java/org/qortal/utils/Blocks.java
View File

@ -53,10 +53,10 @@ public class Blocks {
// all minting group member addresses
List<String> mintingGroupAddresses
= repository.getGroupRepository()
.getGroupMembers(BlockChain.getInstance().getMintingGroupId()).stream()
.map(GroupMemberData::getMember)
.collect(Collectors.toList());
= Groups.getAllMembers(
repository.getGroupRepository(),
Groups.getGroupIdsToMint(BlockChain.getInstance(), blockData.getHeight())
);
// all names, indexed by address
Map<String, String> nameByAddress

122
src/main/java/org/qortal/utils/Groups.java Normal file
View File

@ -0,0 +1,122 @@
package org.qortal.utils;
import org.qortal.block.BlockChain;
import org.qortal.data.group.GroupAdminData;
import org.qortal.data.group.GroupMemberData;
import org.qortal.repository.DataException;
import org.qortal.repository.GroupRepository;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Class Groups
*
* A utility class for group related functionality.
*/
public class Groups {
/**
* Does the member exist in any of these groups?
*
* @param groupRepository the group data repository
* @param groupsIds the group Ids to look for the address
* @param address the address
*
* @return true if the address is in any of the groups listed otherwise false
* @throws DataException
*/
public static boolean memberExistsInAnyGroup(GroupRepository groupRepository, List<Integer> groupsIds, String address) throws DataException {
// if any of the listed groups have the address as a member, then return true
for( Integer groupIdToMint : groupsIds) {
if( groupRepository.memberExists(groupIdToMint, address) ) {
return true;
}
}
// if none of the listed groups have the address as a member, then return false
return false;
}
/**
* Get All Members
*
* Get all the group members from a list of groups.
*
* @param groupRepository the group data repository
* @param groupIds the list of group Ids to look at
*
* @return the list of all members belonging to any of the groups, no duplicates
* @throws DataException
*/
public static List<String> getAllMembers( GroupRepository groupRepository, List<Integer> groupIds ) throws DataException {
// collect all the members in a set, the set keeps out duplicates
Set<String> allMembers = new HashSet<>();
// add all members from each group to the all members set
for( int groupId : groupIds ) {
allMembers.addAll( groupRepository.getGroupMembers(groupId).stream().map(GroupMemberData::getMember).collect(Collectors.toList()));
}
return new ArrayList<>(allMembers);
}
/**
* Get All Admins
*
* Get all the admins from a list of groups.
*
* @param groupRepository the group data repository
* @param groupIds the list of group Ids to look at
*
* @return the list of all admins to any of the groups, no duplicates
* @throws DataException
*/
public static List<String> getAllAdmins( GroupRepository groupRepository, List<Integer> groupIds ) throws DataException {
// collect all the admins in a set, the set keeps out duplicates
Set<String> allAdmins = new HashSet<>();
// collect admins for each group
for( int groupId : groupIds ) {
allAdmins.addAll( groupRepository.getGroupAdmins(groupId).stream().map(GroupAdminData::getAdmin).collect(Collectors.toList()) );
}
return new ArrayList<>(allAdmins);
}
/**
* Get Group Ids To Mint
*
* @param blockchain the blockchain
* @param blockchainHeight the block height to mint
*
* @return the group Ids for the minting groups at the height given
*/
public static List<Integer> getGroupIdsToMint(BlockChain blockchain, int blockchainHeight) {
// sort heights lowest to highest
Comparator<BlockChain.IdsForHeight> compareByHeight = Comparator.comparingInt(entry -> entry.height);
// sort heights highest to lowest
Comparator<BlockChain.IdsForHeight> compareByHeightReversed = compareByHeight.reversed();
// get highest height that is less than the blockchain height
Optional<BlockChain.IdsForHeight> ids = blockchain.getMintingGroupIds().stream()
.filter(entry -> entry.height < blockchainHeight)
.sorted(compareByHeightReversed)
.findFirst();
if( ids.isPresent()) {
return ids.get().ids;
}
else {
return new ArrayList<>(0);
}
}
}

4
src/main/resources/blockchain.json
View File

@ -38,7 +38,9 @@
"blockRewardBatchStartHeight": 1508000,
"blockRewardBatchSize": 1000,
"blockRewardBatchAccountsBlockCount": 25,
"mintingGroupId": 694,
"mintingGroupIds": [
{ "height": 0, "ids": [ 694 ]}
],
"rewardsByHeight": [
{ "height": 1, "reward": 5.00 },
{ "height": 259201, "reward": 4.75 },

14
src/main/resources/q-apps/q-apps.js
View File

@ -84,6 +84,7 @@ isDOMContentLoaded: isDOMContentLoaded ? true : false
function handleQDNResourceDisplayed(pathurl, isDOMContentLoaded) {
// make sure that an empty string the root path
if(pathurl?.startsWith('/render/hash/')) return;
const path = pathurl || '/'
if (!isManualNavigation) {
isManualNavigation = true
@ -284,11 +285,9 @@ window.addEventListener("message", async (event) => {
return;
}
console.log("Core received action: " + JSON.stringify(event.data.action));
let url;
let data = event.data;
let identifier;
switch (data.action) {
case "GET_ACCOUNT_DATA":
return httpGetAsyncWithEvent(event, "/addresses/" + data.address);
@ -383,6 +382,7 @@ window.addEventListener("message", async (event) => {
if (data.identifier != null) url = url.concat("&identifier=" + data.identifier);
if (data.name != null) url = url.concat("&name=" + data.name);
if (data.names != null) data.names.forEach((x, i) => url = url.concat("&name=" + x));
if (data.keywords != null) data.keywords.forEach((x, i) => url = url.concat("&keywords=" + x));
if (data.title != null) url = url.concat("&title=" + data.title);
if (data.description != null) url = url.concat("&description=" + data.description);
if (data.prefix != null) url = url.concat("&prefix=" + new Boolean(data.prefix).toString());
@ -419,7 +419,7 @@ window.addEventListener("message", async (event) => {
return httpGetAsyncWithEvent(event, url);
case "GET_QDN_RESOURCE_PROPERTIES":
let identifier = (data.identifier != null) ? data.identifier : "default";
identifier = (data.identifier != null) ? data.identifier : "default";
url = "/arbitrary/resource/properties/" + data.service + "/" + data.name + "/" + identifier;
return httpGetAsyncWithEvent(event, url);
@ -456,7 +456,7 @@ window.addEventListener("message", async (event) => {
return httpGetAsyncWithEvent(event, url);
case "GET_AT":
url = "/at" + data.atAddress;
url = "/at/" + data.atAddress;
return httpGetAsyncWithEvent(event, url);
case "GET_AT_DATA":
@ -473,7 +473,7 @@ window.addEventListener("message", async (event) => {
case "FETCH_BLOCK":
if (data.signature != null) {
url = "/blocks/" + data.signature;
url = "/blocks/signature/" + data.signature;
} else if (data.height != null) {
url = "/blocks/byheight/" + data.height;
}
@ -694,6 +694,7 @@ const qortalRequestWithTimeout = (request, timeout) =>
* Send current page details to UI
*/
document.addEventListener('DOMContentLoaded', (event) => {
resetVariables()
qortalRequest({
action: "QDN_RESOURCE_DISPLAYED",
@ -712,6 +713,7 @@ resetVariables()
* Handle app navigation
*/
navigation.addEventListener('navigate', (event) => {
const url = new URL(event.destination.url);
let fullpath = url.pathname + url.hash;

54
src/test/java/org/qortal/test/api/CrossChainUtilsTests.java
View File

@ -3,10 +3,15 @@ package org.qortal.test.api;
import org.json.simple.JSONObject;
import org.junit.Assert;
import org.junit.Test;
import org.qortal.api.model.CrossChainTradeLedgerEntry;
import org.qortal.api.resource.CrossChainUtils;
import org.qortal.test.common.ApiCommon;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class CrossChainUtilsTests extends ApiCommon {
@ -137,4 +142,53 @@ public class CrossChainUtilsTests extends ApiCommon {
Assert.assertEquals(5, versionDecimal, 0.001);
Assert.assertFalse(thrown);
}
@Test
public void testWriteToLedgerHeaderOnly() throws IOException {
CrossChainUtils.writeToLedger(new PrintWriter(System.out), new ArrayList<>());
}
@Test
public void testWriteToLedgerOneRow() throws IOException {
CrossChainUtils.writeToLedger(
new PrintWriter(System.out),
List.of(
new CrossChainTradeLedgerEntry(
"QORT",
"LTC",
1000,
0,
"LTC",
1,
System.currentTimeMillis())
)
);
}
@Test
public void testWriteToLedgerTwoRows() throws IOException {
CrossChainUtils.writeToLedger(
new PrintWriter(System.out),
List.of(
new CrossChainTradeLedgerEntry(
"QORT",
"LTC",
1000,
0,
"LTC",
1,
System.currentTimeMillis()
),
new CrossChainTradeLedgerEntry(
"LTC",
"QORT",
1,
0,
"LTC",
1000,
System.currentTimeMillis()
)
)
);
}
}

50
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java
View File

@ -145,56 +145,6 @@ public class ArbitraryDataStorageCapacityTests extends Common {
}
}
@Test
public void testDeleteRandomFilesForName() throws DataException, IOException, InterruptedException, IllegalAccessException {
try (final Repository repository = RepositoryManager.getRepository()) {
String identifier = null; // Not used for this test
Service service = Service.ARBITRARY_DATA;
int chunkSize = 100;
int dataLength = 900; // Actual data length will be longer due to encryption
// Set originalCopyIndicatorFileEnabled to false, otherwise nothing will be deleted as it all originates from this node
FieldUtils.writeField(Settings.getInstance(), "originalCopyIndicatorFileEnabled", false, true);
// Alice hosts some data (with 10 chunks)
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String aliceName = "alice";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), aliceName, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
TransactionUtils.signAndMint(repository, transactionData, alice);
Path alicePath = ArbitraryUtils.generateRandomDataPath(dataLength);
ArbitraryDataFile aliceArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(alice.getPublicKey()), alicePath, aliceName, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize);
// Bob hosts some data too (also with 10 chunks)
PrivateKeyAccount bob = Common.getTestAccount(repository, "bob");
String bobName = "bob";
transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
TransactionUtils.signAndMint(repository, transactionData, bob);
Path bobPath = ArbitraryUtils.generateRandomDataPath(dataLength);
ArbitraryDataFile bobArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(bob.getPublicKey()), bobPath, bobName, identifier, ArbitraryTransactionData.Method.PUT, service, bob, chunkSize);
// All 20 chunks should exist
assertEquals(10, aliceArbitraryDataFile.chunkCount());
assertTrue(aliceArbitraryDataFile.allChunksExist());
assertEquals(10, bobArbitraryDataFile.chunkCount());
assertTrue(bobArbitraryDataFile.allChunksExist());
// Now pretend that Bob has reached his storage limit - this should delete random files
// Run it 10 times to remove the likelihood of the randomizer always picking Alice's files
for (int i=0; i<10; i++) {
ArbitraryDataCleanupManager.getInstance().storageLimitReachedForName(repository, bobName);
}
// Alice should still have all chunks
assertTrue(aliceArbitraryDataFile.allChunksExist());
// Bob should be missing some chunks
assertFalse(bobArbitraryDataFile.allChunksExist());
}
}
private void deleteListsDirectory() {
// Delete lists directory if exists
Path listsPath = Paths.get(Settings.getInstance().getListsPath());

22
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java
View File

@ -73,14 +73,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.FOLLOWED_OR_VIEWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
}
}
@ -108,14 +108,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We shouldn't store or pre-fetch data for this transaction
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
}
}
@ -143,14 +143,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store but not pre-fetch data for this transaction
assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
}
}
@ -178,14 +178,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We should store and pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
}
}
@ -213,14 +213,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We shouldn't store or pre-fetch data for this transaction
assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy());
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We shouldn't store or pre-fetch data for this transaction
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
}
}
@ -236,7 +236,7 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData).isPass());
}
}

48
src/test/java/org/qortal/test/repository/HSQLDBCacheUtilsTests.java
View File

@ -26,6 +26,7 @@ public class HSQLDBCacheUtilsTests {
private static final String DESCRIPTION = "description";
private static final String PREFIX_ONLY = "prefixOnly";
private static final String EXACT_MATCH_NAMES = "exactMatchNames";
private static final String KEYWORDS = "keywords";
private static final String DEFAULT_RESOURCE = "defaultResource";
private static final String MODE = "mode";
private static final String MIN_LEVEL = "minLevel";
@ -299,6 +300,19 @@ public class HSQLDBCacheUtilsTests {
);
}
@Test
public void testAfterNegative() {
ArbitraryResourceData data = new ArbitraryResourceData();
data.created = 10L;
data.name = "Joe";
filterListByMap(
List.of(data),
NAME_LEVEL, new HashMap<>(Map.of(AFTER, 11L)),
0
);
}
@Test
public void testBeforePositive(){
ArbitraryResourceData data = new ArbitraryResourceData();
@ -312,6 +326,19 @@ public class HSQLDBCacheUtilsTests {
);
}
@Test
public void testBeforeNegative(){
ArbitraryResourceData data = new ArbitraryResourceData();
data.created = 10L;
data.name = "Joe";
filterListByMap(
List.of(data),
NAME_LEVEL, new HashMap<>(Map.of(BEFORE, 9L)),
0
);
}
@Test
public void testTitlePositive() {
@ -342,6 +369,25 @@ public class HSQLDBCacheUtilsTests {
);
}
@Test
public void testMetadataNullificationBugSolution(){
ArbitraryResourceData data = new ArbitraryResourceData();
data.metadata = new ArbitraryResourceMetadata();
data.metadata.setDescription("Once upon a time.");
data.name = "Joe";
List<ArbitraryResourceData> list = List.of(data);
filterListByMap(
List.of(data),
NAME_LEVEL, new HashMap<>(Map.of(DESCRIPTION, "Once upon a time.")),
1
);
Assert.assertNotNull(data.metadata);
}
@Test
public void testMinLevelPositive() {
@ -615,6 +661,7 @@ public class HSQLDBCacheUtilsTests {
Optional<String> description = Optional.ofNullable((String) valueByKey.get(DESCRIPTION));
boolean prefixOnly = valueByKey.containsKey(PREFIX_ONLY);
Optional<List<String>> exactMatchNames = Optional.ofNullable((List<String>) valueByKey.get(EXACT_MATCH_NAMES));
Optional<List<String>> keywords = Optional.ofNullable((List<String>) valueByKey.get(KEYWORDS));
boolean defaultResource = valueByKey.containsKey(DEFAULT_RESOURCE);
Optional<SearchMode> mode = Optional.of((SearchMode) valueByKey.getOrDefault(MODE, SearchMode.ALL));
Optional<Integer> minLevel = Optional.ofNullable((Integer) valueByKey.get(MIN_LEVEL));
@ -641,6 +688,7 @@ public class HSQLDBCacheUtilsTests {
description,
prefixOnly,
exactMatchNames,
keywords,
defaultResource,
minLevel,
followedOnly,

102
src/test/java/org/qortal/test/utils/GroupsTestUtils.java Normal file
View File

@ -0,0 +1,102 @@
package org.qortal.test.utils;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.data.transaction.CreateGroupTransactionData;
import org.qortal.data.transaction.GroupInviteTransactionData;
import org.qortal.data.transaction.JoinGroupTransactionData;
import org.qortal.data.transaction.LeaveGroupTransactionData;
import org.qortal.group.Group;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
/**
* Class GroupsTestUtils
*
* Utility methods for testing the Groups class.
*/
public class GroupsTestUtils {
/**
* Create Group
*
* @param repository the data repository
* @param owner the group owner
* @param groupName the group name
* @param isOpen true if the group is public, false for private
*
* @return the group Id
* @throws DataException
*/
public static Integer createGroup(Repository repository, PrivateKeyAccount owner, String groupName, boolean isOpen) throws DataException {
String description = groupName + " (description)";
Group.ApprovalThreshold approvalThreshold = Group.ApprovalThreshold.ONE;
int minimumBlockDelay = 10;
int maximumBlockDelay = 1440;
CreateGroupTransactionData transactionData = new CreateGroupTransactionData(TestTransaction.generateBase(owner), groupName, description, isOpen, approvalThreshold, minimumBlockDelay, maximumBlockDelay);
TransactionUtils.signAndMint(repository, transactionData, owner);
return repository.getGroupRepository().fromGroupName(groupName).getGroupId();
}
/**
* Join Group
*
* @param repository the data repository
* @param joiner the address for the account joining the group
* @param groupId the Id for the group to join
*
* @throws DataException
*/
public static void joinGroup(Repository repository, PrivateKeyAccount joiner, int groupId) throws DataException {
JoinGroupTransactionData transactionData = new JoinGroupTransactionData(TestTransaction.generateBase(joiner), groupId);
TransactionUtils.signAndMint(repository, transactionData, joiner);
}
/**
* Group Invite
*
* @param repository the data repository
* @param admin the admin account to sign the invite
* @param groupId the Id of the group to invite to
* @param invitee the recipient address for the invite
* @param timeToLive the time length of the invite
*
* @throws DataException
*/
public static void groupInvite(Repository repository, PrivateKeyAccount admin, int groupId, String invitee, int timeToLive) throws DataException {
GroupInviteTransactionData transactionData = new GroupInviteTransactionData(TestTransaction.generateBase(admin), groupId, invitee, timeToLive);
TransactionUtils.signAndMint(repository, transactionData, admin);
}
/**
* Leave Group
*
* @param repository the data repository
* @param leaver the account leaving
* @param groupId the Id of the group being left
*
* @throws DataException
*/
public static void leaveGroup(Repository repository, PrivateKeyAccount leaver, int groupId) throws DataException {
LeaveGroupTransactionData transactionData = new LeaveGroupTransactionData(TestTransaction.generateBase(leaver), groupId);
TransactionUtils.signAndMint(repository, transactionData, leaver);
}
/**
* Is Member?
*
* @param repository the data repository
* @param address the account address
* @param groupId the group Id
*
* @return true if the account is a member of the group, otherwise false
* @throws DataException
*/
public static boolean isMember(Repository repository, String address, int groupId) throws DataException {
return repository.getGroupRepository().memberExists(groupId, address);
}
}

199
src/test/java/org/qortal/test/utils/GroupsTests.java Normal file
View File

@ -0,0 +1,199 @@
package org.qortal.test.utils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.block.Block;
import org.qortal.block.BlockChain;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.utils.Groups;
import java.util.List;
import static org.junit.Assert.*;
public class GroupsTests extends Common {
public static final String ALICE = "alice";
public static final String BOB = "bob";
public static final String CHLOE = "chloe";
public static final String DILBERT = "dilbert";
private static final int HEIGHT_1 = 5;
private static final int HEIGHT_2 = 8;
private static final int HEIGHT_3 = 12;
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings();
}
@After
public void afterTest() throws DataException {
Common.orphanCheck();
}
@Test
public void testGetGroupIdsToMintSimple() {
List<Integer> ids = Groups.getGroupIdsToMint(BlockChain.getInstance(), 0);
Assert.assertNotNull(ids);
Assert.assertEquals(0, ids.size());
}
@Test
public void testGetGroupIdsToMintComplex() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
Block block1 = BlockUtils.mintBlocks(repository, HEIGHT_1);
int height1 = block1.getBlockData().getHeight().intValue();
assertEquals(HEIGHT_1 + 1, height1);
List<Integer> ids1 = Groups.getGroupIdsToMint(BlockChain.getInstance(), height1);
Assert.assertEquals(1, ids1.size() );
Assert.assertTrue( ids1.contains( 694 ) );
Block block2 = BlockUtils.mintBlocks(repository, HEIGHT_2 - HEIGHT_1);
int height2 = block2.getBlockData().getHeight().intValue();
assertEquals( HEIGHT_2 + 1, height2);
List<Integer> ids2 = Groups.getGroupIdsToMint(BlockChain.getInstance(), height2);
Assert.assertEquals(2, ids2.size() );
Assert.assertTrue( ids2.contains( 694 ) );
Assert.assertTrue( ids2.contains( 800 ) );
Block block3 = BlockUtils.mintBlocks(repository, HEIGHT_3 - HEIGHT_2);
int height3 = block3.getBlockData().getHeight().intValue();
assertEquals( HEIGHT_3 + 1, height3);
List<Integer> ids3 = Groups.getGroupIdsToMint(BlockChain.getInstance(), height3);
Assert.assertEquals( 1, ids3.size() );
Assert.assertTrue( ids3.contains( 800 ) );
}
}
@Test
public void testMemberExistsInAnyGroupSimple() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
PrivateKeyAccount bob = Common.getTestAccount(repository, "bob");
// Create group
int groupId = GroupsTestUtils.createGroup(repository, alice, "closed-group", false);
// Confirm Bob is not a member
Assert.assertFalse( Groups.memberExistsInAnyGroup(repository.getGroupRepository(), List.of(groupId), bob.getAddress()) );
// Bob to join
GroupsTestUtils.joinGroup(repository, bob, groupId);
// Confirm Bob still not a member
assertFalse(GroupsTestUtils.isMember(repository, bob.getAddress(), groupId));
// Have Alice 'invite' Bob to confirm membership
GroupsTestUtils.groupInvite(repository, alice, groupId, bob.getAddress(), 0); // non-expiring invite
// Confirm Bob now a member
Assert.assertTrue( Groups.memberExistsInAnyGroup(repository.getGroupRepository(), List.of(groupId), bob.getAddress()) );
}
}
@Test
public void testGroupsListedFunctionality() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount alice = Common.getTestAccount(repository, ALICE);
PrivateKeyAccount bob = Common.getTestAccount(repository, BOB);
PrivateKeyAccount chloe = Common.getTestAccount(repository, CHLOE);
PrivateKeyAccount dilbert = Common.getTestAccount(repository, DILBERT);
// Create groups
int group1Id = GroupsTestUtils.createGroup(repository, alice, "group-1", false);
int group2Id = GroupsTestUtils.createGroup(repository, bob, "group-2", false);
// test memberExistsInAnyGroup
Assert.assertTrue(Groups.memberExistsInAnyGroup(repository.getGroupRepository(), List.of(group1Id, group2Id), alice.getAddress()));
Assert.assertFalse(Groups.memberExistsInAnyGroup(repository.getGroupRepository(), List.of(group1Id, group2Id), chloe.getAddress()));
// alice is a member
Assert.assertTrue(GroupsTestUtils.isMember(repository, alice.getAddress(), group1Id));
List<String> allMembersBeforeJoin = Groups.getAllMembers(repository.getGroupRepository(), List.of(group1Id));
// assert one member
Assert.assertNotNull(allMembersBeforeJoin);
Assert.assertEquals(1, allMembersBeforeJoin.size());
List<String> allAdminsBeforeJoin = Groups.getAllAdmins(repository.getGroupRepository(), List.of(group1Id));
// assert one admin
Assert.assertNotNull(allAdminsBeforeJoin);
Assert.assertEquals( 1, allAdminsBeforeJoin.size());
// Bob to join
GroupsTestUtils.joinGroup(repository, bob, group1Id);
// Have Alice 'invite' Bob to confirm membership
GroupsTestUtils.groupInvite(repository, alice, group1Id, bob.getAddress(), 0); // non-expiring invite
List<String> allMembersAfterJoin = Groups.getAllMembers(repository.getGroupRepository(), List.of(group1Id));
// alice and bob are members
Assert.assertNotNull(allMembersAfterJoin);
Assert.assertEquals(2, allMembersAfterJoin.size());
List<String> allAdminsAfterJoin = Groups.getAllAdmins(repository.getGroupRepository(), List.of(group1Id));
// assert still one admin
Assert.assertNotNull(allAdminsAfterJoin);
Assert.assertEquals(1, allAdminsAfterJoin.size());
List<String> allAdminsFor2Groups = Groups.getAllAdmins(repository.getGroupRepository(), List.of(group1Id, group2Id));
// assert 2 admins when including the second group
Assert.assertNotNull(allAdminsFor2Groups);
Assert.assertEquals(2, allAdminsFor2Groups.size());
List<String> allMembersFor2Groups = Groups.getAllMembers(repository.getGroupRepository(), List.of(group1Id, group2Id));
// assert 2 members when including the seconds group
Assert.assertNotNull(allMembersFor2Groups);
Assert.assertEquals(2, allMembersFor2Groups.size());
GroupsTestUtils.leaveGroup(repository, bob, group1Id);
List<String> allMembersForAfterBobLeavesGroup1InAllGroups = Groups.getAllMembers(repository.getGroupRepository(), List.of(group1Id, group2Id));
// alice and bob are members of one group still
Assert.assertNotNull(allMembersForAfterBobLeavesGroup1InAllGroups);
Assert.assertEquals(2, allMembersForAfterBobLeavesGroup1InAllGroups.size());
GroupsTestUtils.groupInvite(repository, alice, group1Id, chloe.getAddress(), 3600);
GroupsTestUtils.groupInvite(repository, bob, group2Id, chloe.getAddress(), 3600);
GroupsTestUtils.joinGroup(repository, chloe, group1Id);
GroupsTestUtils.joinGroup(repository, chloe, group2Id);
List<String> allMembersAfterDilbert = Groups.getAllMembers((repository.getGroupRepository()), List.of(group1Id, group2Id));
// 3 accounts are now members of one group or another
Assert.assertNotNull(allMembersAfterDilbert);
Assert.assertEquals(3, allMembersAfterDilbert.size());
}
}
}

6
src/test/resources/test-chain-v2.json
View File

@ -31,6 +31,12 @@
"blockRewardBatchStartHeight": 999999000,
"blockRewardBatchSize": 10,
"blockRewardBatchAccountsBlockCount": 3,
"mintingGroupIds": [
{ "height": 0, "ids": []},
{ "height": 5, "ids": [694]},
{ "height": 8, "ids": [694, 800]},
{ "height": 12, "ids": [800]}
],
"rewardsByHeight": [
{ "height": 1, "reward": 100 },
{ "height": 11, "reward": 10 },