From 9dba4b29681cbdcb7a7c1ead59311bc5d2865917 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 22 Apr 2023 16:05:12 +0100 Subject: [PATCH 01/34] Initial attempt at a database cache to hold arbitrary resources and metadata. --- .../api/resource/ArbitraryResource.java | 46 +- .../org/qortal/arbitrary/misc/Category.java | 5 +- .../org/qortal/controller/Controller.java | 21 + .../arbitrary/ArbitraryDataManager.java | 36 ++ .../arbitrary/ArbitraryMetadataManager.java | 56 +- ...ceInfo.java => ArbitraryResourceData.java} | 10 +- .../arbitrary/ArbitraryResourceMetadata.java | 47 ++ .../arbitrary/ArbitraryResourceNameInfo.java | 2 +- .../repository/ArbitraryRepository.java | 30 +- .../qortal/repository/RepositoryManager.java | 71 +++ .../hsqldb/HSQLDBArbitraryRepository.java | 478 ++++++++++++++---- .../hsqldb/HSQLDBDatabaseUpdates.java | 40 +- .../transaction/ArbitraryTransaction.java | 130 ++++- .../utils/ArbitraryTransactionUtils.java | 37 +- 14 files changed, 816 insertions(+), 193 deletions(-) rename src/main/java/org/qortal/data/arbitrary/{ArbitraryResourceInfo.java => ArbitraryResourceData.java} (80%) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index c617b517..1e243c51 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -86,12 +86,12 @@ public class ArbitraryResource { "- If default is set to true, only resources without identifiers will be returned.", responses = { @ApiResponse( - content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class)) + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class)) ) } ) @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public List getResources( + public List getResources( @QueryParam("service") Service service, @QueryParam("name") String name, @QueryParam("identifier") String identifier, @@ -133,8 +133,9 @@ public class ArbitraryResource { } } - List resources = repository.getArbitraryRepository() - .getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked, limit, offset, reverse); + List resources = repository.getArbitraryRepository() + .getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked, + includeMetadata, limit, offset, reverse); if (resources == null) { return new ArrayList<>(); @@ -143,9 +144,6 @@ public class ArbitraryResource { if (includeStatus != null && includeStatus) { resources = ArbitraryTransactionUtils.addStatusToResources(resources); } - if (includeMetadata != null && includeMetadata) { - resources = ArbitraryTransactionUtils.addMetadataToResources(resources); - } return resources; @@ -161,12 +159,12 @@ public class ArbitraryResource { "If default is set to true, only resources without identifiers will be returned.", responses = { @ApiResponse( - content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class)) + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class)) ) } ) @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public List searchResources( + public List searchResources( @QueryParam("service") Service service, @Parameter(description = "Query (searches both name and identifier fields)") @QueryParam("query") String query, @Parameter(description = "Identifier (searches identifier field only)") @QueryParam("identifier") String identifier, @@ -206,8 +204,9 @@ public class ArbitraryResource { names = null; } - List resources = repository.getArbitraryRepository() - .searchArbitraryResources(service, query, identifier, names, usePrefixOnly, exactMatchNames, defaultRes, followedOnly, excludeBlocked, limit, offset, reverse); + List resources = repository.getArbitraryRepository() + .searchArbitraryResources(service, query, identifier, names, usePrefixOnly, exactMatchNames, + defaultRes, followedOnly, excludeBlocked, includeMetadata, limit, offset, reverse); if (resources == null) { return new ArrayList<>(); @@ -216,9 +215,6 @@ public class ArbitraryResource { if (includeStatus != null && includeStatus) { resources = ArbitraryTransactionUtils.addStatusToResources(resources); } - if (includeMetadata != null && includeMetadata) { - resources = ArbitraryTransactionUtils.addMetadataToResources(resources); - } return resources; @@ -479,21 +475,20 @@ public class ArbitraryResource { summary = "List arbitrary resources hosted by this node", responses = { @ApiResponse( - content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class)) + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class)) ) } ) @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public List getHostedResources( + public List getHostedResources( @HeaderParam(Security.API_KEY_HEADER) String apiKey, @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus, - @Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata, @Parameter(ref = "limit") @QueryParam("limit") Integer limit, @Parameter(ref = "offset") @QueryParam("offset") Integer offset, @QueryParam("query") String query) { Security.checkApiCallAllowed(request); - List resources = new ArrayList<>(); + List resources = new ArrayList<>(); try (final Repository repository = RepositoryManager.getRepository()) { @@ -509,21 +504,18 @@ public class ArbitraryResource { if (transactionData.getService() == null) { continue; } - ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); - arbitraryResourceInfo.name = transactionData.getName(); - arbitraryResourceInfo.service = transactionData.getService(); - arbitraryResourceInfo.identifier = transactionData.getIdentifier(); - if (!resources.contains(arbitraryResourceInfo)) { - resources.add(arbitraryResourceInfo); + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.name = transactionData.getName(); + arbitraryResourceData.service = transactionData.getService(); + arbitraryResourceData.identifier = transactionData.getIdentifier(); + if (!resources.contains(arbitraryResourceData)) { + resources.add(arbitraryResourceData); } } if (includeStatus != null && includeStatus) { resources = ArbitraryTransactionUtils.addStatusToResources(resources); } - if (includeMetadata != null && includeMetadata) { - resources = ArbitraryTransactionUtils.addMetadataToResources(resources); - } return resources; diff --git a/src/main/java/org/qortal/arbitrary/misc/Category.java b/src/main/java/org/qortal/arbitrary/misc/Category.java index d56e3d5d..3a1489b8 100644 --- a/src/main/java/org/qortal/arbitrary/misc/Category.java +++ b/src/main/java/org/qortal/arbitrary/misc/Category.java @@ -67,9 +67,12 @@ public enum Category { /** * Same as valueOf() but with fallback to UNCATEGORIZED if there's no match * @param name - * @return a Category (using UNCATEGORIZED if no match found) + * @return a Category (using UNCATEGORIZED if no match found), or null if null name passed */ public static Category uncategorizedValueOf(String name) { + if (name == null) { + return null; + } try { return Category.valueOf(name); } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index f0bd1ef5..e4944a66 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -47,6 +47,7 @@ import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.naming.NameData; import org.qortal.data.network.PeerData; +import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ChatTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.event.Event; @@ -400,6 +401,10 @@ public class Controller extends Thread { RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl()); RepositoryManager.setRepositoryFactory(repositoryFactory); RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); + + try (final Repository repository = RepositoryManager.getRepository()) { + RepositoryManager.buildInitialArbitraryResourcesCache(repository); + } } catch (DataException e) { // If exception has no cause then repository is in use by some other process. @@ -891,6 +896,7 @@ public class Controller extends Thread { if (now >= transaction.getDeadline()) { LOGGER.debug(() -> String.format("Deleting expired, unconfirmed transaction %s", Base58.encode(transactionData.getSignature()))); repository.getTransactionRepository().delete(transactionData); + this.onExpiredTransaction(transactionData); deletedCount++; } } @@ -1203,6 +1209,21 @@ public class Controller extends Thread { }); } + /** + * Callback for when we've deleted an expired, unconfirmed transaction. + *

+ * @implSpec performs actions in a new thread + */ + public void onExpiredTransaction(TransactionData transactionData) { + this.callbackExecutor.execute(() -> { + + // If this is an ARBITRARY transaction, we may need to update the cache + if (transactionData.getType() == TransactionType.ARBITRARY) { + ArbitraryDataManager.getInstance().onExpiredArbitraryTransaction((ArbitraryTransactionData)transactionData); + } + }); + } + public void onPeerHandshakeCompleted(Peer peer) { // Only send if outbound if (peer.isOutbound()) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 9284e672..e32fcb0f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -14,6 +14,7 @@ import org.qortal.arbitrary.ArbitraryDataResource; import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; @@ -539,6 +540,41 @@ public class ArbitraryDataManager extends Thread { return true; } + public void onExpiredArbitraryTransaction(ArbitraryTransactionData arbitraryTransactionData) { + if (arbitraryTransactionData.getName() == null) { + // No name, so we don't care about this transaction + return; + } + + Service service = arbitraryTransactionData.getService(); + String name = arbitraryTransactionData.getName(); + String identifier = arbitraryTransactionData.getIdentifier(); + + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.service = service; + arbitraryResourceData.name = name; + arbitraryResourceData.identifier = identifier; + + try (final Repository repository = RepositoryManager.getRepository()) { + // Find next oldest transaction (which is now the latest transaction) + ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(name, service, null, identifier); + + if (latestTransactionData == null) { + // There are no transactions anymore, so we can delete from the cache entirely (this deletes metadata too) + repository.getArbitraryRepository().delete(arbitraryResourceData); + } + else { + // We found the next oldest transaction, so we can update the cache + ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, latestTransactionData); + arbitraryTransaction.updateArbitraryResourceCache(); + arbitraryTransaction.updateArbitraryMetadataCache(); + } +; + } catch (DataException e) { + // Not much we can do, so ignore for now + } + } + public int getPowDifficulty() { return this.powDifficulty; } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java index 663bc22a..a496485b 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java @@ -15,6 +15,7 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; +import org.qortal.transaction.ArbitraryTransaction; import org.qortal.utils.Base58; import org.qortal.utils.ListUtils; import org.qortal.utils.NTP; @@ -324,37 +325,44 @@ public class ArbitraryMetadataManager { Triple newEntry = new Triple<>(null, null, request.getC()); arbitraryMetadataRequests.put(message.getId(), newEntry); - ArbitraryTransactionData arbitraryTransactionData = null; - - // Forwarding - if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) { - - // Get transaction info - try (final Repository repository = RepositoryManager.getRepository()) { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (!(transactionData instanceof ArbitraryTransactionData)) - return; - arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while finding arbitrary transaction metadata for peer %s", peer), e); + // Get transaction info + try (final Repository repository = RepositoryManager.getRepository()) { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (!(transactionData instanceof ArbitraryTransactionData)) { + return; } + ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - // Check if the name is blocked - boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName())); - if (!isBlocked) { - Peer requestingPeer = request.getB(); - if (requestingPeer != null) { + // Forwarding + if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) { - ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile()); - forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId()); + // Check if the name is blocked + boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName())); + if (!isBlocked) { + Peer requestingPeer = request.getB(); + if (requestingPeer != null) { - // Forward to requesting peer - LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer); - if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) { - requestingPeer.disconnect("failed to forward arbitrary metadata"); + ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile()); + forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId()); + + // Forward to requesting peer + LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer); + if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) { + requestingPeer.disconnect("failed to forward arbitrary metadata"); + } } } } + + // Update arbitrary resource caches + if (arbitraryTransactionData != null) { + ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, arbitraryTransactionData); + arbitraryTransaction.updateArbitraryResourceCache(); + arbitraryTransaction.updateArbitraryMetadataCache(); + } + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while finding arbitrary transaction metadata for peer %s", peer), e); } } diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java similarity index 80% rename from src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java rename to src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java index a09fc5ff..4f636177 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java @@ -7,7 +7,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import java.util.Objects; @XmlAccessorType(XmlAccessType.FIELD) -public class ArbitraryResourceInfo { +public class ArbitraryResourceData { public String name; public Service service; @@ -15,11 +15,11 @@ public class ArbitraryResourceInfo { public ArbitraryResourceStatus status; public ArbitraryResourceMetadata metadata; - public Long size; + public Integer size; public Long created; public Long updated; - public ArbitraryResourceInfo() { + public ArbitraryResourceData() { } @Override @@ -32,10 +32,10 @@ public class ArbitraryResourceInfo { if (o == this) return true; - if (!(o instanceof ArbitraryResourceInfo)) + if (!(o instanceof ArbitraryResourceData)) return false; - ArbitraryResourceInfo other = (ArbitraryResourceInfo) o; + ArbitraryResourceData other = (ArbitraryResourceData) o; return Objects.equals(this.name, other.name) && Objects.equals(this.service, other.service) && diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java index a6aa6e26..614a8e69 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java @@ -18,6 +18,9 @@ public class ArbitraryResourceMetadata { private List files; private String mimeType; + // Only included when updating database + private ArbitraryResourceData arbitraryResourceData; + public ArbitraryResourceMetadata() { } @@ -60,4 +63,48 @@ public class ArbitraryResourceMetadata { public List getFiles() { return this.files; } + + public void setTitle(String title) { + this.title = title; + } + + public String getTitle() { + return this.title; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDescription() { + return this.description; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public List getTags() { + return this.tags; + } + + public void setCategory(Category category) { + this.category = category; + + // Also set categoryName + if (category != null) { + this.categoryName = category.getName(); + } + } + + public Category getCategory() { + return this.category; + } + + public void setArbitraryResourceData(ArbitraryResourceData arbitraryResourceData) { + this.arbitraryResourceData = arbitraryResourceData; + } + public ArbitraryResourceData getArbitraryResourceData() { + return this.arbitraryResourceData; + } } diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java index b9be8034..0f91c2c2 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java @@ -9,7 +9,7 @@ import java.util.List; public class ArbitraryResourceNameInfo { public String name; - public List resources = new ArrayList<>(); + public List resources = new ArrayList<>(); public ArbitraryResourceNameInfo() { } diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 9d9ed8ce..63729e46 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -1,9 +1,8 @@ package org.qortal.repository; import org.qortal.arbitrary.misc.Service; -import org.qortal.data.arbitrary.ArbitraryResourceInfo; -import org.qortal.data.arbitrary.ArbitraryResourceNameInfo; -import org.qortal.data.network.ArbitraryPeerData; +import org.qortal.data.arbitrary.ArbitraryResourceData; +import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; @@ -11,23 +10,42 @@ import java.util.List; public interface ArbitraryRepository { + // Utils + public boolean isDataLocal(byte[] signature) throws DataException; public byte[] fetchData(byte[] signature) throws DataException; + + // Transaction related + public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException; public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException; public List getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException; + public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException; + public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException; - public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Integer limit, Integer offset, Boolean reverse) throws DataException; + // Resource related - public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, List namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Integer limit, Integer offset, Boolean reverse) throws DataException; + public ArbitraryResourceData getArbitraryResource(Service service, String name, String identifier) throws DataException; - public List getArbitraryResourceCreatorNames(Service service, String identifier, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException; + public List getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException; + public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException; + + public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, List namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException; + + + // Arbitrary resources cache save/load + + public void save(ArbitraryResourceData arbitraryResourceData) throws DataException; + public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException; + + public void save(ArbitraryResourceMetadata metadata) throws DataException; + public void delete(ArbitraryResourceMetadata metadata) throws DataException; } diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index 9d76ccae..404b6b34 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -2,8 +2,16 @@ package org.qortal.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.api.resource.TransactionsResource; +import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryResourceData; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.settings.Settings; +import org.qortal.transaction.ArbitraryTransaction; +import org.qortal.transaction.Transaction; import java.sql.SQLException; +import java.util.List; import java.util.concurrent.TimeoutException; public abstract class RepositoryManager { @@ -56,6 +64,69 @@ public abstract class RepositoryManager { } } + public static boolean buildInitialArbitraryResourcesCache(Repository repository) throws DataException { + if (Settings.getInstance().isLite()) { + // Lite nodes have no blockchain + return false; + } + + try { + // Check if QDNResources table is empty + List resources = repository.getArbitraryRepository().getArbitraryResources(10, 0, false); + if (!resources.isEmpty()) { + // Resources exist in the cache, so assume complete. + // We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so + // we shouldn't ever be left in a partially rebuilt state. + LOGGER.debug("Arbitrary resources cache already built"); + return false; + } + + LOGGER.info("Building arbitrary resources cache..."); + + final int batchSize = 100; + int offset = 0; + + // Loop through all ARBITRARY transactions, and determine latest state + while (!Controller.isStopping()) { + LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1); + + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false); + if (signatures.isEmpty()) { + // Complete + break; + } + + // Expand signatures to transactions + for (byte[] signature : signatures) { + ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository + .getTransactionRepository().fromSignature(signature); + + if (transactionData.getService() == null) { + // Unsupported service - ignore this resource + continue; + } + + // Update arbitrary resource caches + ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); + arbitraryTransaction.updateArbitraryResourceCache(); + arbitraryTransaction.updateArbitraryMetadataCache(); + } + offset += batchSize; + } + + repository.saveChanges(); + LOGGER.info("Completed build of initial arbitrary resources cache."); + return true; + } + catch (DataException e) { + LOGGER.info("Unable to build initial arbitrary resources cache: {}. The database may have been left in an inconsistent state.", e.getMessage()); + + // Throw an exception so that the node startup is halted, allowing for a retry next time. + repository.discardChanges(); + throw new DataException("Build of initial arbitrary resources cache failed."); + } + } + public static void setRequestedCheckpoint(Boolean quick) { quickCheckpointRequested = quick; } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 87841ca9..9c88e39c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -2,10 +2,10 @@ package org.qortal.repository.hsqldb; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.bouncycastle.util.Longs; +import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Service; -import org.qortal.data.arbitrary.ArbitraryResourceInfo; -import org.qortal.data.arbitrary.ArbitraryResourceNameInfo; +import org.qortal.data.arbitrary.ArbitraryResourceData; +import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.data.transaction.BaseTransactionData; @@ -22,6 +22,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; public class HSQLDBArbitraryRepository implements ArbitraryRepository { @@ -41,6 +42,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { return (ArbitraryTransactionData) transactionData; } + + // Utils + @Override public boolean isDataLocal(byte[] signature) throws DataException { ArbitraryTransactionData transactionData = getTransactionData(signature); @@ -113,6 +117,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { return null; } + + // Transaction related + @Override public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException { // Already hashed? Nothing to do @@ -211,8 +218,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } } - @Override - public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException { + private ArbitraryTransactionData getSingleTransaction(String name, Service service, Method method, String identifier, boolean firstNotLast) throws DataException { StringBuilder sql = new StringBuilder(1024); sql.append("SELECT type, reference, signature, creator, created_when, fee, " + @@ -228,7 +234,16 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { sql.append(method.value); } - sql.append("ORDER BY created_when DESC LIMIT 1"); + sql.append(" ORDER BY created_when"); + + if (firstNotLast) { + sql.append(" ASC"); + } + else { + sql.append(" DESC"); + } + + sql.append(" LIMIT 1"); try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value, identifier, identifier)) { if (resultSet == null) @@ -284,13 +299,189 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } @Override - public List getArbitraryResources(Service service, String identifier, List names, - boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, - Integer limit, Integer offset, Boolean reverse) throws DataException { + public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException { + return this.getSingleTransaction(name, service, method, identifier, true); + } + + @Override + public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException { + return this.getSingleTransaction(name, service, method, identifier, false); + } + + + // Resource related + + @Override + public ArbitraryResourceData getArbitraryResource(Service service, String name, String identifier) throws DataException { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); - sql.append("SELECT name, service, identifier, MAX(size) AS max_size FROM ArbitraryTransactions WHERE 1=1"); + // Name is required + if (name == null) { + return null; + } + + sql.append("SELECT name, service, identifier, size, created_when, updated_when, " + + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + + "FROM ArbitraryResourcesCache " + + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + + "WHERE service = ? AND name = ?"); + + bindParams.add(service.value); + bindParams.add(name); + + if (identifier != null) { + sql.append(" AND identifier = ?"); + bindParams.add(identifier); + } + else { + sql.append(" AND identifier IS NULL"); + } + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return null; + + String nameResult = resultSet.getString(1); + Service serviceResult = Service.valueOf(resultSet.getInt(2)); + String identifierResult = resultSet.getString(3); + Integer sizeResult = resultSet.getInt(4); + Long created = resultSet.getLong(5); + Long updated = resultSet.getLong(6); + + // Optional metadata fields + String title = resultSet.getString(7); + String description = resultSet.getString(8); + String category = resultSet.getString(9); + String tag1 = resultSet.getString(10); + String tag2 = resultSet.getString(11); + String tag3 = resultSet.getString(12); + String tag4 = resultSet.getString(13); + String tag5 = resultSet.getString(14); + + if (Objects.equals(identifierResult, "default")) { + // Map "default" back to null. This is optional but probably less confusing than returning "default". + identifierResult = null; + } + + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.name = nameResult; + arbitraryResourceData.service = serviceResult; + arbitraryResourceData.identifier = identifierResult; + arbitraryResourceData.size = sizeResult; + arbitraryResourceData.created = created; + arbitraryResourceData.updated = (updated == 0) ? null : updated; + + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setTitle(title); + metadata.setDescription(description); + metadata.setCategory(Category.uncategorizedValueOf(category)); + + List tags = new ArrayList<>(); + if (tag1 != null) tags.add(tag1); + if (tag2 != null) tags.add(tag2); + if (tag3 != null) tags.add(tag3); + if (tag4 != null) tags.add(tag4); + if (tag5 != null) tags.add(tag5); + metadata.setTags(!tags.isEmpty() ? tags : null); + + arbitraryResourceData.metadata = metadata; + + return arbitraryResourceData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary resource from repository", e); + } + } + @Override + public List getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException { + StringBuilder sql = new StringBuilder(512); + List bindParams = new ArrayList<>(); + + sql.append("SELECT name, service, identifier, size, created_when, updated_when, " + + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + + "FROM ArbitraryResourcesCache " + + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + + "WHERE name IS NOT NULL ORDER BY created_when"); + + if (reverse != null && reverse) { + sql.append(" DESC"); + } + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List arbitraryResources = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return arbitraryResources; + + do { + String nameResult = resultSet.getString(1); + Service serviceResult = Service.valueOf(resultSet.getInt(2)); + String identifierResult = resultSet.getString(3); + Integer sizeResult = resultSet.getInt(4); + Long created = resultSet.getLong(5); + Long updated = resultSet.getLong(6); + + // Optional metadata fields + String title = resultSet.getString(7); + String description = resultSet.getString(8); + String category = resultSet.getString(9); + String tag1 = resultSet.getString(10); + String tag2 = resultSet.getString(11); + String tag3 = resultSet.getString(12); + String tag4 = resultSet.getString(13); + String tag5 = resultSet.getString(14); + + if (Objects.equals(identifierResult, "default")) { + // Map "default" back to null. This is optional but probably less confusing than returning "default". + identifierResult = null; + } + + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.name = nameResult; + arbitraryResourceData.service = serviceResult; + arbitraryResourceData.identifier = identifierResult; + arbitraryResourceData.size = sizeResult; + arbitraryResourceData.created = created; + arbitraryResourceData.updated = (updated == 0) ? null : updated; + + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setTitle(title); + metadata.setDescription(description); + metadata.setCategory(Category.uncategorizedValueOf(category)); + + List tags = new ArrayList<>(); + if (tag1 != null) tags.add(tag1); + if (tag2 != null) tags.add(tag2); + if (tag3 != null) tags.add(tag3); + if (tag4 != null) tags.add(tag4); + if (tag5 != null) tags.add(tag5); + metadata.setTags(!tags.isEmpty() ? tags : null); + + arbitraryResourceData.metadata = metadata; + + arbitraryResources.add(arbitraryResourceData); + } while (resultSet.next()); + + return arbitraryResources; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary resources from repository", e); + } + } + + @Override + public List getArbitraryResources(Service service, String identifier, List names, + boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, + Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException { + StringBuilder sql = new StringBuilder(512); + List bindParams = new ArrayList<>(); + + sql.append("SELECT name, service, identifier, size, created_when, updated_when, " + + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + + "FROM ArbitraryResourcesCache " + + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + + "WHERE name IS NOT NULL"); if (service != null) { sql.append(" AND service = "); @@ -351,7 +542,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } } - sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); + sql.append(" ORDER BY created_when"); if (reverse != null && reverse) { sql.append(" DESC"); @@ -359,49 +550,82 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { HSQLDBRepository.limitOffsetSql(sql, limit, offset); - List arbitraryResources = new ArrayList<>(); + List arbitraryResources = new ArrayList<>(); try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { if (resultSet == null) - return null; + return arbitraryResources; do { String nameResult = resultSet.getString(1); Service serviceResult = Service.valueOf(resultSet.getInt(2)); String identifierResult = resultSet.getString(3); Integer sizeResult = resultSet.getInt(4); + Long created = resultSet.getLong(5); + Long updated = resultSet.getLong(6); - // We should filter out resources without names - if (nameResult == null) { - continue; + // Optional metadata fields + String title = resultSet.getString(7); + String description = resultSet.getString(8); + String category = resultSet.getString(9); + String tag1 = resultSet.getString(10); + String tag2 = resultSet.getString(11); + String tag3 = resultSet.getString(12); + String tag4 = resultSet.getString(13); + String tag5 = resultSet.getString(14); + + if (Objects.equals(identifierResult, "default")) { + // Map "default" back to null. This is optional but probably less confusing than returning "default". + identifierResult = null; } - ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); - arbitraryResourceInfo.name = nameResult; - arbitraryResourceInfo.service = serviceResult; - arbitraryResourceInfo.identifier = identifierResult; - arbitraryResourceInfo.size = Longs.valueOf(sizeResult); + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.name = nameResult; + arbitraryResourceData.service = serviceResult; + arbitraryResourceData.identifier = identifierResult; + arbitraryResourceData.size = sizeResult; + arbitraryResourceData.created = created; + arbitraryResourceData.updated = (updated == 0) ? null : updated; - arbitraryResources.add(arbitraryResourceInfo); + if (includeMetadata != null && includeMetadata) { + // TODO: we could avoid the join altogether + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setTitle(title); + metadata.setDescription(description); + metadata.setCategory(Category.uncategorizedValueOf(category)); + + List tags = new ArrayList<>(); + if (tag1 != null) tags.add(tag1); + if (tag2 != null) tags.add(tag2); + if (tag3 != null) tags.add(tag3); + if (tag4 != null) tags.add(tag4); + if (tag5 != null) tags.add(tag5); + metadata.setTags(!tags.isEmpty() ? tags : null); + + arbitraryResourceData.metadata = metadata; + } + + arbitraryResources.add(arbitraryResourceData); } while (resultSet.next()); return arbitraryResources; } catch (SQLException e) { - throw new DataException("Unable to fetch arbitrary transactions from repository", e); + throw new DataException("Unable to fetch arbitrary resources from repository", e); } } @Override - public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, + public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, List exactMatchNames, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, - Integer limit, Integer offset, Boolean reverse) throws DataException { + Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); - sql.append("SELECT name, service, identifier, MAX(size) AS max_size, MIN(created_when) AS date_created, MAX(created_when) AS date_updated " + - "FROM ArbitraryTransactions " + - "JOIN Transactions USING (signature) " + - "WHERE 1=1"); + sql.append("SELECT name, service, identifier, size, created_when, updated_when, " + + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + + "FROM ArbitraryResourcesCache " + + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + + "WHERE name IS NOT NULL"); if (service != null) { sql.append(" AND service = "); @@ -492,7 +716,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } } - sql.append(" GROUP BY name, service, identifier ORDER BY date_created"); + sql.append(" ORDER BY created_when"); if (reverse != null && reverse) { sql.append(" DESC"); @@ -500,98 +724,156 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { HSQLDBRepository.limitOffsetSql(sql, limit, offset); - List arbitraryResources = new ArrayList<>(); + List arbitraryResources = new ArrayList<>(); try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { if (resultSet == null) - return null; + return arbitraryResources; do { String nameResult = resultSet.getString(1); Service serviceResult = Service.valueOf(resultSet.getInt(2)); String identifierResult = resultSet.getString(3); Integer sizeResult = resultSet.getInt(4); - long dateCreated = resultSet.getLong(5); - long dateUpdated = resultSet.getLong(6); + Long created = resultSet.getLong(5); + Long updated = resultSet.getLong(6); - // We should filter out resources without names - if (nameResult == null) { - continue; + // Optional metadata fields + String title = resultSet.getString(7); + String description = resultSet.getString(8); + String category = resultSet.getString(9); + String tag1 = resultSet.getString(10); + String tag2 = resultSet.getString(11); + String tag3 = resultSet.getString(12); + String tag4 = resultSet.getString(13); + String tag5 = resultSet.getString(14); + + if (Objects.equals(identifierResult, "default")) { + // Map "default" back to null. This is optional but probably less confusing than returning "default". + identifierResult = null; } - ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); - arbitraryResourceInfo.name = nameResult; - arbitraryResourceInfo.service = serviceResult; - arbitraryResourceInfo.identifier = identifierResult; - arbitraryResourceInfo.size = Longs.valueOf(sizeResult); - arbitraryResourceInfo.created = dateCreated; - arbitraryResourceInfo.updated = dateUpdated; + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.name = nameResult; + arbitraryResourceData.service = serviceResult; + arbitraryResourceData.identifier = identifierResult; + arbitraryResourceData.size = sizeResult; + arbitraryResourceData.created = created; + arbitraryResourceData.updated = (updated == 0) ? null : updated; - arbitraryResources.add(arbitraryResourceInfo); + if (includeMetadata != null && includeMetadata) { + // TODO: we could avoid the join altogether + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setTitle(title); + metadata.setDescription(description); + metadata.setCategory(Category.uncategorizedValueOf(category)); + + List tags = new ArrayList<>(); + if (tag1 != null) tags.add(tag1); + if (tag2 != null) tags.add(tag2); + if (tag3 != null) tags.add(tag3); + if (tag4 != null) tags.add(tag4); + if (tag5 != null) tags.add(tag5); + metadata.setTags(!tags.isEmpty() ? tags : null); + + arbitraryResourceData.metadata = metadata; + } + + arbitraryResources.add(arbitraryResourceData); } while (resultSet.next()); return arbitraryResources; } catch (SQLException e) { - throw new DataException("Unable to fetch arbitrary transactions from repository", e); + throw new DataException("Unable to fetch arbitrary resources from repository", e); + } + } + + + // Arbitrary resources cache save/load + + @Override + public void save(ArbitraryResourceData arbitraryResourceData) throws DataException { + HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryResourcesCache"); + + saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name) + .bind("identifier", arbitraryResourceData.identifier).bind("size", arbitraryResourceData.size) + .bind("created_when", arbitraryResourceData.created).bind("updated_when", arbitraryResourceData.updated); + + try { + saveHelper.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save arbitrary resource info into repository", e); } } @Override - public List getArbitraryResourceCreatorNames(Service service, String identifier, - boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException { - StringBuilder sql = new StringBuilder(512); + public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException { + // NOTE: arbitrary metadata are deleted automatically by the database thanks to "ON DELETE CASCADE" + // in ArbitraryMetadataCache' FOREIGN KEY definition. + try { + this.repository.delete("ArbitraryResourcesCache", "service = ? AND name = ? AND identifier = ?", + arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier); - sql.append("SELECT name FROM ArbitraryTransactions WHERE 1=1"); - - if (service != null) { - sql.append(" AND service = "); - sql.append(service.value); - } - - if (defaultResource) { - // Default resource requested - use NULL identifier - // The AND ? IS NULL AND ? IS NULL is a hack to make use of the identifier params in checkedExecute() - identifier = null; - sql.append(" AND (identifier IS NULL AND ? IS NULL AND ? IS NULL)"); - } - else { - // Non-default resource requested - // Use an exact match identifier, or list all if supplied identifier is null - sql.append(" AND (identifier = ? OR (? IS NULL))"); - } - - sql.append(" GROUP BY name ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); - - if (reverse != null && reverse) { - sql.append(" DESC"); - } - - HSQLDBRepository.limitOffsetSql(sql, limit, offset); - - List arbitraryResources = new ArrayList<>(); - - try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), identifier, identifier)) { - if (resultSet == null) - return null; - - do { - String name = resultSet.getString(1); - - // We should filter out resources without names - if (name == null) { - continue; - } - - ArbitraryResourceNameInfo arbitraryResourceNameInfo = new ArbitraryResourceNameInfo(); - arbitraryResourceNameInfo.name = name; - - arbitraryResources.add(arbitraryResourceNameInfo); - } while (resultSet.next()); - - return arbitraryResources; } catch (SQLException e) { - throw new DataException("Unable to fetch arbitrary transactions from repository", e); + throw new DataException("Unable to delete account from repository", e); } } + + /* Arbitrary metadata cache */ + + @Override + public void save(ArbitraryResourceMetadata metadata) throws DataException { + HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryMetadataCache"); + + ArbitraryResourceData arbitraryResourceData = metadata.getArbitraryResourceData(); + if (arbitraryResourceData == null) { + throw new DataException("Can't save metadata without a referenced resource"); + } + + String tag1 = null; + String tag2 = null; + String tag3 = null; + String tag4 = null; + String tag5 = null; + + List tags = metadata.getTags(); + if (tags != null) { + if (tags.size() > 0) tag1 = tags.get(0); + if (tags.size() > 1) tag2 = tags.get(1); + if (tags.size() > 2) tag3 = tags.get(2); + if (tags.size() > 3) tag4 = tags.get(3); + if (tags.size() > 4) tag5 = tags.get(4); + } + + String category = metadata.getCategory() != null ? metadata.getCategory().toString() : null; + + saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name) + .bind("identifier", arbitraryResourceData.identifier).bind("title", metadata.getTitle()) + .bind("description", metadata.getDescription()).bind("category", category) + .bind("tag1", tag1).bind("tag2", tag2).bind("tag3", tag3).bind("tag4", tag4) + .bind("tag5", tag5); + + try { + saveHelper.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save arbitrary metadata into repository", e); + } + } + + @Override + public void delete(ArbitraryResourceMetadata metadata) throws DataException { + ArbitraryResourceData arbitraryResourceData = metadata.getArbitraryResourceData(); + if (arbitraryResourceData == null) { + throw new DataException("Can't delete metadata without a referenced resource"); + } + + try { + this.repository.delete("ArbitraryMetadataCache", "service = ? AND name = ? AND identifier = ?", + arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier); + + } catch (SQLException e) { + throw new DataException("Unable to delete account from repository", e); + } + } } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index aecac034..e3c705e8 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -901,7 +901,7 @@ public class HSQLDBDatabaseUpdates { case 37: // ARBITRARY transaction updates for off-chain data storage - // We may want to use a nonce rather than a transaction fee on the data chain + // We may want to use a nonce rather than a transaction fee for ARBITRARY transactions stmt.execute("ALTER TABLE ArbitraryTransactions ADD nonce INT NOT NULL DEFAULT 0"); // We need to know the total size of the data file(s) associated with each transaction stmt.execute("ALTER TABLE ArbitraryTransactions ADD size INT NOT NULL DEFAULT 0"); @@ -993,6 +993,44 @@ public class HSQLDBDatabaseUpdates { stmt.execute("ALTER TABLE CancelSellNameTransactions ADD sale_price QortalAmount"); break; + case 47: + // We need to keep a local cache of arbitrary resources (items published to QDN), for easier searching. + // IMPORTANT: this is a cache of the last known state of a resource (both confirmed + // and valid unconfirmed). It cannot be assumed that all nodes will contain the same state at a + // given block height, and therefore must NOT be used for any consensus/validation code. It is + // simply a cache, to avoid having to query the raw transactions and the metadata in flat files + // when serving API requests. + // ARBITRARY transactions aren't really suitable for updating resources in the same way we'd update + // names or groups for instance, as there is no distinction between creations and updates, and metadata + // is off-chain. Plus, QDN allows (valid) unconfirmed data to be queried and viewed. It is very + // easy to keep a cache of the latest transaction's data, but anything more than that would need + // considerable thought (and most likely a rewrite). + + stmt.execute("CREATE TABLE ArbitraryResourcesCache (service SMALLINT NOT NULL, " + + "name RegisteredName NOT NULL, identifier VARCHAR(64), size INT NOT NULL, " + + "created_when EpochMillis NOT NULL, updated_when EpochMillis, " + + "PRIMARY KEY (service, name, identifier))"); + // For finding resources by service. + stmt.execute("CREATE INDEX ArbitraryResourcesServiceIndex ON ArbitraryResourcesCache (service)"); + // For finding resources by name. + stmt.execute("CREATE INDEX ArbitraryResourcesNameIndex ON ArbitraryResourcesCache (name)"); + // For finding resources by identifier. + stmt.execute("CREATE INDEX ArbitraryResourcesIdentifierIndex ON ArbitraryResourcesCache (identifier)"); + // For finding resources by creation date (the default column when ordering). + stmt.execute("CREATE INDEX ArbitraryResourcesCreatedIndex ON ArbitraryResourcesCache (created_when)"); + // Use a separate table space as this table will be very large. + stmt.execute("SET TABLE ArbitraryResourcesCache NEW SPACE"); + + stmt.execute("CREATE TABLE ArbitraryMetadataCache (service SMALLINT NOT NULL, " + + "name RegisteredName NOT NULL, identifier VARCHAR(64), " + + "title VARCHAR(80), description VARCHAR(240), category VARCHAR(64), " + + "tag1 VARCHAR(20), tag2 VARCHAR(20), tag3 VARCHAR(20), tag4 VARCHAR(20), tag5 VARCHAR(20), " + + "PRIMARY KEY (service, name, identifier), FOREIGN KEY (service, name, identifier) " + + "REFERENCES ArbitraryResourcesCache (service, name, identifier) ON DELETE CASCADE)"); + // For finding metadata by title. + stmt.execute("CREATE INDEX ArbitraryMetadataTitleIndex ON ArbitraryMetadataCache (title)"); + break; + default: // nothing to do return false; diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 7034d7b8..88be95a2 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -1,18 +1,21 @@ package org.qortal.transaction; -import java.util.Arrays; +import java.io.IOException; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; import org.qortal.account.Account; +import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; +import org.qortal.arbitrary.misc.Service; import org.qortal.block.BlockChain; import org.qortal.controller.arbitrary.ArbitraryDataManager; -import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.crypto.Crypto; import org.qortal.crypto.MemoryPoW; import org.qortal.data.PaymentData; +import org.qortal.data.arbitrary.ArbitraryResourceData; +import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; @@ -248,6 +251,10 @@ public class ArbitraryTransaction extends Transaction { ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData); } } + + // Add to arbitrary resource caches + this.updateArbitraryResourceCache(); + this.updateArbitraryMetadataCache(); } @Override @@ -304,4 +311,123 @@ public class ArbitraryTransaction extends Transaction { return null; } + /** + * Update the arbitrary resources cache. + * This finds the latest transaction and replaces the + * majority of the data in the cache. The current + * transaction is used for the created time, + * if it has a lower timestamp than the existing value. + * It's also used to identify the correct + * service/name/identifier combination. + * + * @throws DataException + */ + public void updateArbitraryResourceCache() throws DataException { + // Don't cache resources without a name (such as auto updates) + if (arbitraryTransactionData.getName() == null) { + return; + } + + // Get the latest transaction + ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); + if (latestTransactionData == null) { + // We don't have a latest transaction, so give up + return; + } + + Service service = arbitraryTransactionData.getService(); + String name = arbitraryTransactionData.getName(); + String identifier = arbitraryTransactionData.getIdentifier(); + + // In the cache we store null identifiers as "default", as it is part of the primary key + if (identifier == null) { + identifier = "default"; + } + + // Get existing cached entry if it exists + ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository() + .getArbitraryResource(service, name, identifier); + + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.service = service; + arbitraryResourceData.name = name; + arbitraryResourceData.identifier = identifier; + + // Check for existing cached data + if (existingArbitraryResourceData == null) { + // Nothing exists yet, so set everything from the newest transaction + arbitraryResourceData.created = latestTransactionData.getTimestamp(); + arbitraryResourceData.updated = null; + } + else { + // An entry already exists - update created time from current transaction if this is older + arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp()); + + // Set updated time to the latest transaction's timestamp, unless it matches the creation time + if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) { + // Latest transaction matches created time, so it hasn't been updated + arbitraryResourceData.updated = null; + } + else { + arbitraryResourceData.updated = latestTransactionData.getTimestamp(); + } + } + + arbitraryResourceData.size = latestTransactionData.getSize(); + + // Save + repository.getArbitraryRepository().save(arbitraryResourceData); + } + + public void updateArbitraryMetadataCache() throws DataException { + // Get the latest transaction + ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); + if (latestTransactionData == null) { + // We don't have a latest transaction, so give up + return; + } + + Service service = latestTransactionData.getService(); + String name = latestTransactionData.getName(); + String identifier = latestTransactionData.getIdentifier(); + + // In the cache we store null identifiers as "default", as it is part of the primary key + if (identifier == null) { + identifier = "default"; + } + + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.service = service; + arbitraryResourceData.name = name; + arbitraryResourceData.identifier = identifier; + + // Update metadata for latest transaction if it is local + if (latestTransactionData.getMetadataHash() != null) { + ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature()); + if (metadataFile.exists()) { + ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath()); + try { + transactionMetadata.read(); + + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setArbitraryResourceData(arbitraryResourceData); + metadata.setTitle(transactionMetadata.getTitle()); + metadata.setDescription(transactionMetadata.getDescription()); + metadata.setCategory(transactionMetadata.getCategory()); + metadata.setTags(transactionMetadata.getTags()); + repository.getArbitraryRepository().save(metadata); + + } catch (IOException e) { + // Ignore, as we can add it again later + } + } else { + // We don't have a local copy of this metadata file, so delete it from the cache + // It will be re-added if the file later arrives via the network + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setArbitraryResourceData(arbitraryResourceData); + repository.getArbitraryRepository().delete(metadata); + } + } + } + } diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index efd84110..15ade240 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -4,10 +4,8 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.*; -import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Service; -import org.qortal.data.arbitrary.ArbitraryResourceInfo; -import org.qortal.data.arbitrary.ArbitraryResourceMetadata; +import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceStatus; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; @@ -258,8 +256,7 @@ public class ArbitraryTransactionUtils { "chunks if needed", Base58.encode(completeHash)); ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, cleanupAfter); - } - else { + } else { // File might be in use. It's best to leave it and it it will be cleaned up later. } } @@ -271,6 +268,7 @@ public class ArbitraryTransactionUtils { * When first uploaded, files go into a _misc folder as they are not yet associated with a * transaction signature. Once the transaction is broadcast, they need to be moved to the * correct location, keyed by the transaction signature. + * * @param arbitraryTransactionData * @return * @throws DataException @@ -356,8 +354,7 @@ public class ArbitraryTransactionUtils { file.createNewFile(); } } - } - catch (DataException | IOException e) { + } catch (DataException | IOException e) { LOGGER.info("Unable to check and relocate all files for signature {}: {}", Base58.encode(arbitraryTransactionData.getSignature()), e.getMessage()); } @@ -366,7 +363,7 @@ public class ArbitraryTransactionUtils { } public static List limitOffsetTransactions(List transactions, - Integer limit, Integer offset) { + Integer limit, Integer offset) { if (limit != null && limit == 0) { limit = null; } @@ -389,6 +386,7 @@ public class ArbitraryTransactionUtils { /** * Lookup status of resource + * * @param service * @param name * @param identifier @@ -413,10 +411,10 @@ public class ArbitraryTransactionUtils { return resource.getStatus(false); } - public static List addStatusToResources(List resources) { + public static List addStatusToResources(List resources) { // Determine and add the status of each resource - List updatedResources = new ArrayList<>(); - for (ArbitraryResourceInfo resourceInfo : resources) { + List updatedResources = new ArrayList<>(); + for (ArbitraryResourceData resourceInfo : resources) { try { ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME, resourceInfo.service, resourceInfo.identifier); @@ -433,21 +431,4 @@ public class ArbitraryTransactionUtils { } return updatedResources; } - - public static List addMetadataToResources(List resources) { - // Add metadata fields to each resource if they exist - List updatedResources = new ArrayList<>(); - for (ArbitraryResourceInfo resourceInfo : resources) { - ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME, - resourceInfo.service, resourceInfo.identifier); - ArbitraryDataTransactionMetadata transactionMetadata = resource.getLatestTransactionMetadata(); - ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(transactionMetadata, false); - if (resourceMetadata != null) { - resourceInfo.metadata = resourceMetadata; - } - updatedResources.add(resourceInfo); - } - return updatedResources; - } - } From eb7a29dd2eefa2cbf748140cd54eedc6ca55f848 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Apr 2023 10:58:27 +0100 Subject: [PATCH 02/34] Fixed bugs. --- .../arbitrary/ArbitraryDataManager.java | 1 + .../arbitrary/ArbitraryMetadataManager.java | 1 + .../hsqldb/HSQLDBArbitraryRepository.java | 20 ++++++++++++++----- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index e32fcb0f..60bf63b3 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -570,6 +570,7 @@ public class ArbitraryDataManager extends Thread { arbitraryTransaction.updateArbitraryMetadataCache(); } ; + repository.saveChanges(); } catch (DataException e) { // Not much we can do, so ignore for now } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java index a496485b..f99ec953 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java @@ -359,6 +359,7 @@ public class ArbitraryMetadataManager { ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, arbitraryTransactionData); arbitraryTransaction.updateArbitraryResourceCache(); arbitraryTransaction.updateArbitraryMetadataCache(); + repository.saveChanges(); } } catch (DataException e) { diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 9c88e39c..2ee49fa0 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -2,6 +2,7 @@ package org.qortal.repository.hsqldb; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Service; import org.qortal.data.arbitrary.ArbitraryResourceData; @@ -219,6 +220,11 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } private ArbitraryTransactionData getSingleTransaction(String name, Service service, Method method, String identifier, boolean firstNotLast) throws DataException { + if (name == null || service == null) { + // Required fields + return null; + } + StringBuilder sql = new StringBuilder(1024); sql.append("SELECT type, reference, signature, creator, created_when, fee, " + @@ -490,7 +496,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { if (defaultResource) { // Default resource requested - use NULL identifier - sql.append(" AND identifier IS NULL"); + sql.append(" AND identifier='default'"); } else { // Non-default resource requested @@ -641,7 +647,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { if (defaultResource) { // Default resource requested - use NULL identifier and search name only - sql.append(" AND LCASE(name) LIKE ? AND identifier IS NULL"); + sql.append(" AND LCASE(name) LIKE ? AND identifier='default'"); bindParams.add(queryWildcard); } else { // Non-default resource requested @@ -831,13 +837,17 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { throw new DataException("Can't save metadata without a referenced resource"); } + // Trim metadata values if they are too long to fit in the db + String title = ArbitraryDataTransactionMetadata.limitTitle(metadata.getTitle()); + String description = ArbitraryDataTransactionMetadata.limitTitle(metadata.getDescription()); + List tags = ArbitraryDataTransactionMetadata.limitTags(metadata.getTags()); + String tag1 = null; String tag2 = null; String tag3 = null; String tag4 = null; String tag5 = null; - List tags = metadata.getTags(); if (tags != null) { if (tags.size() > 0) tag1 = tags.get(0); if (tags.size() > 1) tag2 = tags.get(1); @@ -849,8 +859,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { String category = metadata.getCategory() != null ? metadata.getCategory().toString() : null; saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name) - .bind("identifier", arbitraryResourceData.identifier).bind("title", metadata.getTitle()) - .bind("description", metadata.getDescription()).bind("category", category) + .bind("identifier", arbitraryResourceData.identifier).bind("title", title) + .bind("description", description).bind("category", category) .bind("tag1", tag1).bind("tag2", tag2).bind("tag3", tag3).bind("tag4", tag4) .bind("tag5", tag5); From 200b0f3412867da03ba2dc8375d64136b3e6467a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 7 May 2023 11:43:48 +0100 Subject: [PATCH 03/34] Added `POST /arbitrary/resources/cache/rebuild` endpoint to allow a rebuild of the cache. --- .../api/resource/ArbitraryResource.java | 30 +++++++++++++++++++ .../org/qortal/controller/Controller.java | 2 +- .../qortal/repository/RepositoryManager.java | 4 +-- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 1e243c51..9a2fceaa 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1119,6 +1119,36 @@ public class ArbitraryResource { } + @POST + @Path("/resources/cache/rebuild") + @Operation( + summary = "Rebuild arbitrary resources cache from transactions", + responses = { + @ApiResponse( + description = "true on success", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "boolean" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String rebuildCache(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + RepositoryManager.buildArbitraryResourcesCache(repository, true); + + return "true"; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage()); + } + } + + // Shared methods private String preview(String directoryPath, Service service) { diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index e4944a66..1de6f776 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -403,7 +403,7 @@ public class Controller extends Thread { RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); try (final Repository repository = RepositoryManager.getRepository()) { - RepositoryManager.buildInitialArbitraryResourcesCache(repository); + RepositoryManager.buildArbitraryResourcesCache(repository, false); } } catch (DataException e) { diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index 404b6b34..e4a287c0 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -64,7 +64,7 @@ public abstract class RepositoryManager { } } - public static boolean buildInitialArbitraryResourcesCache(Repository repository) throws DataException { + public static boolean buildArbitraryResourcesCache(Repository repository, boolean forceRebuild) throws DataException { if (Settings.getInstance().isLite()) { // Lite nodes have no blockchain return false; @@ -73,7 +73,7 @@ public abstract class RepositoryManager { try { // Check if QDNResources table is empty List resources = repository.getArbitraryRepository().getArbitraryResources(10, 0, false); - if (!resources.isEmpty()) { + if (!resources.isEmpty() && !forceRebuild) { // Resources exist in the cache, so assume complete. // We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so // we shouldn't ever be left in a partially rebuilt state. From 94f4c501fa1fe4aaf3b69383d5e86dc7cf345341 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 7 May 2023 11:49:57 +0100 Subject: [PATCH 04/34] Update caches where possible when processing arbitrary transactions. --- .../transaction/ArbitraryTransaction.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 88be95a2..2a3c78af 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -5,6 +5,8 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.qortal.account.Account; import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Service; @@ -32,6 +34,8 @@ import org.qortal.utils.NTP; public class ArbitraryTransaction extends Transaction { + private static final Logger LOGGER = LogManager.getLogger(ArbitraryTransaction.class); + // Properties private ArbitraryTransactionData arbitraryTransactionData; @@ -274,6 +278,30 @@ public class ArbitraryTransaction extends Transaction { public void process() throws DataException { // Wrap and delegate payment processing to Payment class. new Payment(this.repository).process(arbitraryTransactionData.getSenderPublicKey(), arbitraryTransactionData.getPayments()); + + // Update caches + this.updateCaches(); + } + + private void updateCaches() { + try { + // If the data is local, we need to perform a few actions + if (isDataLocal()) { + + // We have the data for this transaction, so invalidate the file cache + if (arbitraryTransactionData.getName() != null) { + ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData); + } + } + + // Add/update arbitrary resource caches + this.updateArbitraryResourceCache(); + this.updateArbitraryMetadataCache(); + + } catch (Exception e) { + // Log and ignore all exceptions. The cache is updated from other places too, and can be rebuilt if needed. + LOGGER.info("Unable to update arbitrary caches", e); + } } @Override From c0f29f848ff74232f797e9a4822acbda1453fd67 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 7 May 2023 12:27:51 +0100 Subject: [PATCH 05/34] Fixed more bugs. --- .../arbitrary/ArbitraryResourceMetadata.java | 4 ++++ .../org/qortal/repository/RepositoryManager.java | 6 +++--- .../hsqldb/HSQLDBArbitraryRepository.java | 16 ++++++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java index 614a8e69..c6f0ae62 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java @@ -101,6 +101,10 @@ public class ArbitraryResourceMetadata { return this.category; } + public boolean hasMetadata() { + return title != null || description != null || tags != null || category != null || files != null || mimeType != null; + } + public void setArbitraryResourceData(ArbitraryResourceData arbitraryResourceData) { this.arbitraryResourceData = arbitraryResourceData; } diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index e4a287c0..d7608401 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -115,15 +115,15 @@ public abstract class RepositoryManager { } repository.saveChanges(); - LOGGER.info("Completed build of initial arbitrary resources cache."); + LOGGER.info("Completed build of arbitrary resources cache."); return true; } catch (DataException e) { - LOGGER.info("Unable to build initial arbitrary resources cache: {}. The database may have been left in an inconsistent state.", e.getMessage()); + LOGGER.info("Unable to build arbitrary resources cache: {}. The database may have been left in an inconsistent state.", e.getMessage()); // Throw an exception so that the node startup is halted, allowing for a retry next time. repository.discardChanges(); - throw new DataException("Build of initial arbitrary resources cache failed."); + throw new DataException("Build of arbitrary resources cache failed."); } } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 2ee49fa0..05633e0a 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -391,7 +391,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { if (tag5 != null) tags.add(tag5); metadata.setTags(!tags.isEmpty() ? tags : null); - arbitraryResourceData.metadata = metadata; + if (metadata.hasMetadata()) { + arbitraryResourceData.metadata = metadata; + } return arbitraryResourceData; } catch (SQLException e) { @@ -465,7 +467,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { if (tag5 != null) tags.add(tag5); metadata.setTags(!tags.isEmpty() ? tags : null); - arbitraryResourceData.metadata = metadata; + if (metadata.hasMetadata()) { + arbitraryResourceData.metadata = metadata; + } arbitraryResources.add(arbitraryResourceData); } while (resultSet.next()); @@ -608,7 +612,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { if (tag5 != null) tags.add(tag5); metadata.setTags(!tags.isEmpty() ? tags : null); - arbitraryResourceData.metadata = metadata; + if (metadata.hasMetadata()) { + arbitraryResourceData.metadata = metadata; + } } arbitraryResources.add(arbitraryResourceData); @@ -782,7 +788,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { if (tag5 != null) tags.add(tag5); metadata.setTags(!tags.isEmpty() ? tags : null); - arbitraryResourceData.metadata = metadata; + if (metadata.hasMetadata()) { + arbitraryResourceData.metadata = metadata; + } } arbitraryResources.add(arbitraryResourceData); From 865d3d8aff7120d36799bdfca46e16649b9bf1c4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 7 May 2023 12:46:01 +0100 Subject: [PATCH 06/34] Fixed ordering, to keep consistency with existing approach. --- .../org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 05633e0a..3061b7b0 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -552,7 +552,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } } - sql.append(" ORDER BY created_when"); + sql.append(" ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); if (reverse != null && reverse) { sql.append(" DESC"); From 961aa9eefd2ffe48adf8f3027a06327f40d4f4eb Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 7 May 2023 13:02:22 +0100 Subject: [PATCH 07/34] Show splash screen when building QDN cache. --- src/main/java/org/qortal/repository/RepositoryManager.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index d7608401..26eb49e3 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -6,6 +6,7 @@ import org.qortal.api.resource.TransactionsResource; import org.qortal.controller.Controller; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.gui.SplashFrame; import org.qortal.settings.Settings; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; @@ -82,6 +83,7 @@ public abstract class RepositoryManager { } LOGGER.info("Building arbitrary resources cache..."); + SplashFrame.getInstance().updateStatus("Building QDN cache - please wait..."); final int batchSize = 100; int offset = 0; From d03a2d7da931b5f460a4b5c587bc31d09d846c1f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 7 May 2023 15:50:19 +0100 Subject: [PATCH 08/34] Resource statuses moved to the db, so they don't have to be calculated on demand for every API call. --- .../api/gateway/resource/GatewayResource.java | 2 +- .../api/resource/ArbitraryResource.java | 17 +-- .../ArbitraryDataBuildQueueItem.java | 4 + .../qortal/arbitrary/ArbitraryDataReader.java | 2 +- .../arbitrary/ArbitraryDataResource.java | 32 +++-- .../PirateChainWalletController.java | 2 +- .../arbitrary/ArbitraryDataBuilderThread.java | 12 ++ .../data/arbitrary/ArbitraryResourceData.java | 21 ++++ .../arbitrary/ArbitraryResourceStatus.java | 39 ++++-- .../repository/ArbitraryRepository.java | 2 + .../hsqldb/HSQLDBArbitraryRepository.java | 112 +++++++++++------- .../hsqldb/HSQLDBDatabaseUpdates.java | 2 +- .../transaction/ArbitraryTransaction.java | 8 ++ .../utils/ArbitraryTransactionUtils.java | 25 +--- 14 files changed, 176 insertions(+), 104 deletions(-) diff --git a/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java b/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java index 9c77753f..7d76be3d 100644 --- a/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java +++ b/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java @@ -70,7 +70,7 @@ public class GatewayResource { } ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); - return resource.getStatus(false); + return resource.getStatus(); } diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 9a2fceaa..2b3b986f 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -141,10 +141,6 @@ public class ArbitraryResource { return new ArrayList<>(); } - if (includeStatus != null && includeStatus) { - resources = ArbitraryTransactionUtils.addStatusToResources(resources); - } - return resources; } catch (DataException e) { @@ -212,10 +208,6 @@ public class ArbitraryResource { return new ArrayList<>(); } - if (includeStatus != null && includeStatus) { - resources = ArbitraryTransactionUtils.addStatusToResources(resources); - } - return resources; } catch (DataException e) { @@ -243,7 +235,7 @@ public class ArbitraryResource { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorizationOrApiKey(request, name, service, null, apiKey); - return ArbitraryTransactionUtils.getStatus(service, name, null, build); + return ArbitraryTransactionUtils.getStatus(service, name, null, build, true); } @GET @@ -290,7 +282,7 @@ public class ArbitraryResource { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, apiKey); - return ArbitraryTransactionUtils.getStatus(service, name, identifier, build); + return ArbitraryTransactionUtils.getStatus(service, name, identifier, build, true); } @@ -482,7 +474,6 @@ public class ArbitraryResource { @ApiErrors({ApiError.REPOSITORY_ISSUE}) public List getHostedResources( @HeaderParam(Security.API_KEY_HEADER) String apiKey, - @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus, @Parameter(ref = "limit") @QueryParam("limit") Integer limit, @Parameter(ref = "offset") @QueryParam("offset") Integer offset, @QueryParam("query") String query) { @@ -513,10 +504,6 @@ public class ArbitraryResource { } } - if (includeStatus != null && includeStatus) { - resources = ArbitraryTransactionUtils.addStatusToResources(resources); - } - return resources; } catch (DataException e) { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java index 4a02f092..465d1391 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -4,6 +4,7 @@ import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.arbitrary.misc.Service; import org.qortal.repository.DataException; +import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.NTP; import java.io.IOException; @@ -51,6 +52,9 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource { arbitraryDataReader.loadSynchronously(true); } finally { this.buildEndTimestamp = NTP.getTime(); + + // Update status after build + ArbitraryTransactionUtils.getStatus(service, resourceId, identifier, false, true); } } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index b9e62e56..f281cec5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -240,7 +240,7 @@ public class ArbitraryDataReader { try { Files.createDirectories(this.workingPath); } catch (IOException e) { - throw new DataException("Unable to create temp directory"); + throw new DataException(String.format("Unable to create temp directory %s: %s", this.workingPath, e.getMessage())); } } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index a4650dfc..4aed21fc 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -9,6 +9,7 @@ import org.qortal.arbitrary.misc.Service; import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; +import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceStatus; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.repository.DataException; @@ -57,17 +58,34 @@ public class ArbitraryDataResource { this.identifier = identifier; } - public ArbitraryResourceStatus getStatus(boolean quick) { - // Calculate the chunk counts - // Avoid this for "quick" statuses, to speed things up - if (!quick) { - this.calculateChunkCounts(); + public ArbitraryResourceStatus getStatusAndUpdateCache(boolean updateCache) { + ArbitraryResourceStatus arbitraryResourceStatus = this.getStatus(); - if (!this.exists) { - return new ArbitraryResourceStatus(Status.NOT_PUBLISHED, this.localChunkCount, this.totalChunkCount); + if (updateCache) { + // Update cache if possible + ArbitraryResourceStatus.Status status = arbitraryResourceStatus != null ? arbitraryResourceStatus.getStatus() : null; + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(this.service, this.resourceId, this.identifier); + + try (final Repository repository = RepositoryManager.getRepository()) { + repository.getArbitraryRepository().setStatus(arbitraryResourceData, status); + repository.saveChanges(); + + } catch (DataException e) { + LOGGER.info("Unable to update status cache for resource {}: {}", arbitraryResourceData, e.getMessage()); } } + return arbitraryResourceStatus; + } + + public ArbitraryResourceStatus getStatus() { + // Calculate the chunk counts + this.calculateChunkCounts(); + + if (!this.exists) { + return new ArbitraryResourceStatus(Status.NOT_PUBLISHED, this.localChunkCount, this.totalChunkCount); + } + if (resourceIdType != ResourceIdType.NAME) { // We only support statuses for resources with a name return new ArbitraryResourceStatus(Status.UNSUPPORTED, this.localChunkCount, this.totalChunkCount); diff --git a/src/main/java/org/qortal/controller/PirateChainWalletController.java b/src/main/java/org/qortal/controller/PirateChainWalletController.java index 90e65329..e009d531 100644 --- a/src/main/java/org/qortal/controller/PirateChainWalletController.java +++ b/src/main/java/org/qortal/controller/PirateChainWalletController.java @@ -187,7 +187,7 @@ public class PirateChainWalletController extends Thread { // Check its status ArbitraryResourceStatus status = ArbitraryTransactionUtils.getStatus( - t.getService(), t.getName(), t.getIdentifier(), false); + t.getService(), t.getName(), t.getIdentifier(), false, true); if (status.getStatus() != ArbitraryResourceStatus.Status.READY) { LOGGER.info("Not ready yet: {}", status.getTitle()); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java index 0fb685a3..a2165a9e 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java @@ -5,13 +5,17 @@ import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryResourceStatus; import org.qortal.repository.DataException; +import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.NTP; import java.io.IOException; import java.util.Comparator; import java.util.Map; +import static org.qortal.data.arbitrary.ArbitraryResourceStatus.Status.*; + public class ArbitraryDataBuilderThread implements Runnable { @@ -69,6 +73,14 @@ public class ArbitraryDataBuilderThread implements Runnable { continue; } + // Get status before build + ArbitraryResourceStatus arbitraryResourceStatus = ArbitraryTransactionUtils.getStatus(queueItem.getService(), queueItem.getResourceId(), queueItem.getIdentifier(), false, true); + if (arbitraryResourceStatus.getStatus() == NOT_PUBLISHED) { + // No point in building a non-existent resource + this.removeFromQueue(queueItem); + continue; + } + // Set the start timestamp, to prevent other threads from building it at the same time queueItem.prepareForBuild(); } diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java index 4f636177..ffd30209 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java @@ -6,6 +6,8 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import java.util.Objects; +import static org.qortal.data.arbitrary.ArbitraryResourceStatus.Status; + @XmlAccessorType(XmlAccessType.FIELD) public class ArbitraryResourceData { @@ -22,11 +24,30 @@ public class ArbitraryResourceData { public ArbitraryResourceData() { } + public ArbitraryResourceData(Service service, String name, String identifier) { + if (identifier == null) { + identifier = "default"; + } + + this.service = service; + this.name = name; + this.identifier = identifier; + } + @Override public String toString() { return String.format("%s %s %s", name, service, identifier); } + public void setStatus(Status status) { + if (status == null) { + this.status = null; + } + else { + this.status = new ArbitraryResourceStatus(status); + } + } + @Override public boolean equals(Object o) { if (o == this) diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java index 54dd2af6..6513776a 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java @@ -2,29 +2,46 @@ package org.qortal.data.arbitrary; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Map; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; @XmlAccessorType(XmlAccessType.FIELD) public class ArbitraryResourceStatus { public enum Status { - PUBLISHED("Published", "Published but not yet downloaded"), - NOT_PUBLISHED("Not published", "Resource does not exist"), - DOWNLOADING("Downloading", "Locating and downloading files..."), - DOWNLOADED("Downloaded", "Files downloaded"), - BUILDING("Building", "Building..."), - READY("Ready", "Ready"), - MISSING_DATA("Missing data", "Unable to locate all files. Please try again later"), - BUILD_FAILED("Build failed", "Build failed. Please try again later"), - UNSUPPORTED("Unsupported", "Unsupported request"), - BLOCKED("Blocked", "Name is blocked so content cannot be served"); + // Note: integer values must not be updated, as they are stored in the db + PUBLISHED(1, "Published", "Published but not yet downloaded"), + NOT_PUBLISHED(2, "Not published", "Resource does not exist"), + DOWNLOADING(3, "Downloading", "Locating and downloading files..."), + DOWNLOADED(4, "Downloaded", "Files downloaded"), + BUILDING(5, "Building", "Building..."), + READY(6, "Ready", "Ready"), + MISSING_DATA(7, "Missing data", "Unable to locate all files. Please try again later"), + BUILD_FAILED(8, "Build failed", "Build failed. Please try again later"), + UNSUPPORTED(9, "Unsupported", "Unsupported request"), + BLOCKED(10, "Blocked", "Name is blocked so content cannot be served"); + public int value; private String title; private String description; - Status(String title, String description) { + private static final Map map = stream(Status.values()) + .collect(toMap(status -> status.value, status -> status)); + + Status(int value, String title, String description) { + this.value = value; this.title = title; this.description = description; } + + public static Status valueOf(Integer value) { + if (value == null) { + return null; + } + return map.get(value); + } } private Status status; diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 63729e46..7a21a1a2 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -3,6 +3,7 @@ package org.qortal.repository; import org.qortal.arbitrary.misc.Service; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceMetadata; +import org.qortal.data.arbitrary.ArbitraryResourceStatus; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; @@ -44,6 +45,7 @@ public interface ArbitraryRepository { // Arbitrary resources cache save/load public void save(ArbitraryResourceData arbitraryResourceData) throws DataException; + public void setStatus(ArbitraryResourceData arbitraryResourceData, ArbitraryResourceStatus.Status status) throws DataException; public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException; public void save(ArbitraryResourceMetadata metadata) throws DataException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 3061b7b0..5564d1d3 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -7,6 +7,7 @@ import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Service; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceMetadata; +import org.qortal.data.arbitrary.ArbitraryResourceStatus; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.data.transaction.BaseTransactionData; @@ -327,7 +328,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { return null; } - sql.append("SELECT name, service, identifier, size, created_when, updated_when, " + + sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + "FROM ArbitraryResourcesCache " + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + @@ -352,18 +353,19 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { Service serviceResult = Service.valueOf(resultSet.getInt(2)); String identifierResult = resultSet.getString(3); Integer sizeResult = resultSet.getInt(4); - Long created = resultSet.getLong(5); - Long updated = resultSet.getLong(6); + Integer status = resultSet.getInt(5); + Long created = resultSet.getLong(6); + Long updated = resultSet.getLong(7); // Optional metadata fields - String title = resultSet.getString(7); - String description = resultSet.getString(8); - String category = resultSet.getString(9); - String tag1 = resultSet.getString(10); - String tag2 = resultSet.getString(11); - String tag3 = resultSet.getString(12); - String tag4 = resultSet.getString(13); - String tag5 = resultSet.getString(14); + String title = resultSet.getString(8); + String description = resultSet.getString(9); + String category = resultSet.getString(10); + String tag1 = resultSet.getString(11); + String tag2 = resultSet.getString(12); + String tag3 = resultSet.getString(13); + String tag4 = resultSet.getString(14); + String tag5 = resultSet.getString(15); if (Objects.equals(identifierResult, "default")) { // Map "default" back to null. This is optional but probably less confusing than returning "default". @@ -375,6 +377,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { arbitraryResourceData.service = serviceResult; arbitraryResourceData.identifier = identifierResult; arbitraryResourceData.size = sizeResult; + arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status)); arbitraryResourceData.created = created; arbitraryResourceData.updated = (updated == 0) ? null : updated; @@ -405,7 +408,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); - sql.append("SELECT name, service, identifier, size, created_when, updated_when, " + + sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + "FROM ArbitraryResourcesCache " + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + @@ -428,18 +431,19 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { Service serviceResult = Service.valueOf(resultSet.getInt(2)); String identifierResult = resultSet.getString(3); Integer sizeResult = resultSet.getInt(4); - Long created = resultSet.getLong(5); - Long updated = resultSet.getLong(6); + Integer status = resultSet.getInt(5); + Long created = resultSet.getLong(6); + Long updated = resultSet.getLong(7); // Optional metadata fields - String title = resultSet.getString(7); - String description = resultSet.getString(8); - String category = resultSet.getString(9); - String tag1 = resultSet.getString(10); - String tag2 = resultSet.getString(11); - String tag3 = resultSet.getString(12); - String tag4 = resultSet.getString(13); - String tag5 = resultSet.getString(14); + String title = resultSet.getString(8); + String description = resultSet.getString(9); + String category = resultSet.getString(10); + String tag1 = resultSet.getString(11); + String tag2 = resultSet.getString(12); + String tag3 = resultSet.getString(13); + String tag4 = resultSet.getString(14); + String tag5 = resultSet.getString(15); if (Objects.equals(identifierResult, "default")) { // Map "default" back to null. This is optional but probably less confusing than returning "default". @@ -451,6 +455,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { arbitraryResourceData.service = serviceResult; arbitraryResourceData.identifier = identifierResult; arbitraryResourceData.size = sizeResult; + arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status)); arbitraryResourceData.created = created; arbitraryResourceData.updated = (updated == 0) ? null : updated; @@ -487,7 +492,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); - sql.append("SELECT name, service, identifier, size, created_when, updated_when, " + + sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + "FROM ArbitraryResourcesCache " + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + @@ -571,18 +576,19 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { Service serviceResult = Service.valueOf(resultSet.getInt(2)); String identifierResult = resultSet.getString(3); Integer sizeResult = resultSet.getInt(4); - Long created = resultSet.getLong(5); - Long updated = resultSet.getLong(6); + Integer status = resultSet.getInt(5); + Long created = resultSet.getLong(6); + Long updated = resultSet.getLong(7); // Optional metadata fields - String title = resultSet.getString(7); - String description = resultSet.getString(8); - String category = resultSet.getString(9); - String tag1 = resultSet.getString(10); - String tag2 = resultSet.getString(11); - String tag3 = resultSet.getString(12); - String tag4 = resultSet.getString(13); - String tag5 = resultSet.getString(14); + String title = resultSet.getString(8); + String description = resultSet.getString(9); + String category = resultSet.getString(10); + String tag1 = resultSet.getString(11); + String tag2 = resultSet.getString(12); + String tag3 = resultSet.getString(13); + String tag4 = resultSet.getString(14); + String tag5 = resultSet.getString(15); if (Objects.equals(identifierResult, "default")) { // Map "default" back to null. This is optional but probably less confusing than returning "default". @@ -594,6 +600,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { arbitraryResourceData.service = serviceResult; arbitraryResourceData.identifier = identifierResult; arbitraryResourceData.size = sizeResult; + arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status)); arbitraryResourceData.created = created; arbitraryResourceData.updated = (updated == 0) ? null : updated; @@ -633,7 +640,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); - sql.append("SELECT name, service, identifier, size, created_when, updated_when, " + + sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + "FROM ArbitraryResourcesCache " + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + @@ -747,18 +754,19 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { Service serviceResult = Service.valueOf(resultSet.getInt(2)); String identifierResult = resultSet.getString(3); Integer sizeResult = resultSet.getInt(4); - Long created = resultSet.getLong(5); - Long updated = resultSet.getLong(6); + Integer status = resultSet.getInt(5); + Long created = resultSet.getLong(6); + Long updated = resultSet.getLong(7); // Optional metadata fields - String title = resultSet.getString(7); - String description = resultSet.getString(8); - String category = resultSet.getString(9); - String tag1 = resultSet.getString(10); - String tag2 = resultSet.getString(11); - String tag3 = resultSet.getString(12); - String tag4 = resultSet.getString(13); - String tag5 = resultSet.getString(14); + String title = resultSet.getString(8); + String description = resultSet.getString(9); + String category = resultSet.getString(10); + String tag1 = resultSet.getString(11); + String tag2 = resultSet.getString(12); + String tag3 = resultSet.getString(13); + String tag4 = resultSet.getString(14); + String tag5 = resultSet.getString(15); if (Objects.equals(identifierResult, "default")) { // Map "default" back to null. This is optional but probably less confusing than returning "default". @@ -770,6 +778,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { arbitraryResourceData.service = serviceResult; arbitraryResourceData.identifier = identifierResult; arbitraryResourceData.size = sizeResult; + arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status)); arbitraryResourceData.created = created; arbitraryResourceData.updated = (updated == 0) ? null : updated; @@ -809,6 +818,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { public void save(ArbitraryResourceData arbitraryResourceData) throws DataException { HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryResourcesCache"); + // "status" isn't saved here as we update this field separately saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name) .bind("identifier", arbitraryResourceData.identifier).bind("size", arbitraryResourceData.size) .bind("created_when", arbitraryResourceData.created).bind("updated_when", arbitraryResourceData.updated); @@ -820,6 +830,20 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } } + @Override + public void setStatus(ArbitraryResourceData arbitraryResourceData, ArbitraryResourceStatus.Status status) throws DataException { + if (status == null) { + return; + } + String updateSql = "UPDATE ArbitraryResourcesCache SET status = ? WHERE service = ? AND name = ? AND identifier = ?"; + + try { + this.repository.executeCheckedUpdate(updateSql, status.value, arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier); + } catch (SQLException e) { + throw new DataException("Unable to set status for arbitrary resource", e); + } + } + @Override public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException { // NOTE: arbitrary metadata are deleted automatically by the database thanks to "ON DELETE CASCADE" diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index e3c705e8..b54f4b08 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -1008,7 +1008,7 @@ public class HSQLDBDatabaseUpdates { stmt.execute("CREATE TABLE ArbitraryResourcesCache (service SMALLINT NOT NULL, " + "name RegisteredName NOT NULL, identifier VARCHAR(64), size INT NOT NULL, " - + "created_when EpochMillis NOT NULL, updated_when EpochMillis, " + + "status INTEGER, created_when EpochMillis NOT NULL, updated_when EpochMillis, " + "PRIMARY KEY (service, name, identifier))"); // For finding resources by service. stmt.execute("CREATE INDEX ArbitraryResourcesServiceIndex ON ArbitraryResourcesCache (service)"); diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 2a3c78af..b2954dd9 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -8,6 +8,7 @@ import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.account.Account; +import org.qortal.arbitrary.ArbitraryDataResource; import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Service; import org.qortal.block.BlockChain; @@ -18,6 +19,7 @@ import org.qortal.crypto.MemoryPoW; import org.qortal.data.PaymentData; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceMetadata; +import org.qortal.data.arbitrary.ArbitraryResourceStatus; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; @@ -405,6 +407,12 @@ public class ArbitraryTransaction extends Transaction { // Save repository.getArbitraryRepository().save(arbitraryResourceData); + + // Update status + ArbitraryDataResource resource = new ArbitraryDataResource(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); + ArbitraryResourceStatus arbitraryResourceStatus = resource.getStatus(); + ArbitraryResourceStatus.Status status = arbitraryResourceStatus != null ? arbitraryResourceStatus.getStatus() : null; + repository.getArbitraryRepository().setStatus(arbitraryResourceData, status); } public void updateArbitraryMetadataCache() throws DataException { diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index 15ade240..bf7e2aa6 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -393,7 +393,7 @@ public class ArbitraryTransactionUtils { * @param build * @return */ - public static ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build) { + public static ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build, boolean updateCache) { // If "build" has been specified, build the resource before returning its status if (build != null && build == true) { @@ -408,27 +408,6 @@ public class ArbitraryTransactionUtils { } ArbitraryDataResource resource = new ArbitraryDataResource(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); - return resource.getStatus(false); - } - - public static List addStatusToResources(List resources) { - // Determine and add the status of each resource - List updatedResources = new ArrayList<>(); - for (ArbitraryResourceData resourceInfo : resources) { - try { - ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME, - resourceInfo.service, resourceInfo.identifier); - ArbitraryResourceStatus status = resource.getStatus(true); - if (status != null) { - resourceInfo.status = status; - } - updatedResources.add(resourceInfo); - - } catch (Exception e) { - // Catch and log all exceptions, since some systems are experiencing 500 errors when including statuses - LOGGER.info("Caught exception when adding status to resource {}: {}", resourceInfo, e.toString()); - } - } - return updatedResources; + return resource.getStatusAndUpdateCache(updateCache); } } From cdcb268bd9ae45977edd43ef0da0c7f3a94eaa9d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 7 May 2023 17:21:41 +0100 Subject: [PATCH 09/34] Exclude status if includeStatus != true --- .../qortal/api/resource/ArbitraryResource.java | 4 ++-- .../qortal/repository/ArbitraryRepository.java | 4 ++-- .../hsqldb/HSQLDBArbitraryRepository.java | 15 +++++++++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 2b3b986f..08a9b73c 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -135,7 +135,7 @@ public class ArbitraryResource { List resources = repository.getArbitraryRepository() .getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked, - includeMetadata, limit, offset, reverse); + includeMetadata, includeStatus, limit, offset, reverse); if (resources == null) { return new ArrayList<>(); @@ -202,7 +202,7 @@ public class ArbitraryResource { List resources = repository.getArbitraryRepository() .searchArbitraryResources(service, query, identifier, names, usePrefixOnly, exactMatchNames, - defaultRes, followedOnly, excludeBlocked, includeMetadata, limit, offset, reverse); + defaultRes, followedOnly, excludeBlocked, includeMetadata, includeStatus, limit, offset, reverse); if (resources == null) { return new ArrayList<>(); diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 7a21a1a2..572d2a59 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -37,9 +37,9 @@ public interface ArbitraryRepository { public List getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException; - public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException; + public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; - public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, List namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException; + public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, List namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; // Arbitrary resources cache save/load diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 5564d1d3..71428cb6 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -488,7 +488,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { @Override public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, - Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException { + Boolean includeMetadata, Boolean includeStatus, + Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); @@ -600,10 +601,13 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { arbitraryResourceData.service = serviceResult; arbitraryResourceData.identifier = identifierResult; arbitraryResourceData.size = sizeResult; - arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status)); arbitraryResourceData.created = created; arbitraryResourceData.updated = (updated == 0) ? null : updated; + if (includeStatus != null && includeStatus) { + arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status)); + } + if (includeMetadata != null && includeMetadata) { // TODO: we could avoid the join altogether ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); @@ -636,7 +640,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { @Override public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, List exactMatchNames, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, - Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException { + Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); @@ -778,10 +782,13 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { arbitraryResourceData.service = serviceResult; arbitraryResourceData.identifier = identifierResult; arbitraryResourceData.size = sizeResult; - arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status)); arbitraryResourceData.created = created; arbitraryResourceData.updated = (updated == 0) ? null : updated; + if (includeStatus != null && includeStatus) { + arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status)); + } + if (includeMetadata != null && includeMetadata) { // TODO: we could avoid the join altogether ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); From 2fd5bfb11a320607a7c087595f9d3274cefb52a1 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 7 May 2023 17:57:14 +0100 Subject: [PATCH 10/34] Support title/description metadata searching in `GET /arbitrary/resources/search` "query" searches name, identifier, title and description fields "title" searches title only "description" searches description only All support "&prefix=true", to indicate searching by prefix only. --- .../api/resource/ArbitraryResource.java | 6 ++-- .../repository/ArbitraryRepository.java | 2 +- .../hsqldb/HSQLDBArbitraryRepository.java | 31 ++++++++++++++----- src/main/resources/q-apps/q-apps.js | 2 ++ 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 08a9b73c..8762a7f1 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -162,9 +162,11 @@ public class ArbitraryResource { @ApiErrors({ApiError.REPOSITORY_ISSUE}) public List searchResources( @QueryParam("service") Service service, - @Parameter(description = "Query (searches both name and identifier fields)") @QueryParam("query") String query, + @Parameter(description = "Query (searches name, identifier, title and description fields)") @QueryParam("query") String query, @Parameter(description = "Identifier (searches identifier field only)") @QueryParam("identifier") String identifier, @Parameter(description = "Name (searches name field only)") @QueryParam("name") List names, + @Parameter(description = "Title (searches title metadata field only)") @QueryParam("title") String title, + @Parameter(description = "Description (searches description metadata field only)") @QueryParam("description") String description, @Parameter(description = "Prefix only (if true, only the beginning of fields are matched)") @QueryParam("prefix") Boolean prefixOnly, @Parameter(description = "Exact match names only (if true, partial name matches are excluded)") @QueryParam("exactmatchnames") Boolean exactMatchNamesOnly, @Parameter(description = "Default resources (without identifiers) only") @QueryParam("default") Boolean defaultResource, @@ -201,7 +203,7 @@ public class ArbitraryResource { } List resources = repository.getArbitraryRepository() - .searchArbitraryResources(service, query, identifier, names, usePrefixOnly, exactMatchNames, + .searchArbitraryResources(service, query, identifier, names, title, description, usePrefixOnly, exactMatchNames, defaultRes, followedOnly, excludeBlocked, includeMetadata, includeStatus, limit, offset, reverse); if (resources == null) { diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 572d2a59..590aa3b8 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -39,7 +39,7 @@ public interface ArbitraryRepository { public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; - public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, List namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; + public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, List namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; // Arbitrary resources cache save/load diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 71428cb6..45706e11 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -638,7 +638,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } @Override - public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, + public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, List exactMatchNames, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(512); @@ -669,9 +669,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } else { // Non-default resource requested // In this case we search the identifier as well as the name - sql.append(" AND (LCASE(name) LIKE ? OR LCASE(identifier) LIKE ?)"); - bindParams.add(queryWildcard); - bindParams.add(queryWildcard); + sql.append(" AND (LCASE(name) LIKE ? OR LCASE(identifier) LIKE ? OR LCASE(title) LIKE ? OR LCASE(description) LIKE ?)"); + bindParams.add(queryWildcard); bindParams.add(queryWildcard); bindParams.add(queryWildcard); bindParams.add(queryWildcard); } } @@ -683,6 +682,22 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { bindParams.add(queryWildcard); } + // Handle title metadata matches + if (title != null) { + // Search anywhere in the title, unless "prefixOnly" has been requested + String queryWildcard = prefixOnly ? String.format("%s%%", title.toLowerCase()) : String.format("%%%s%%", title.toLowerCase()); + sql.append(" AND LCASE(title) LIKE ?"); + bindParams.add(queryWildcard); + } + + // Handle description metadata matches + if (description != null) { + // Search anywhere in the description, unless "prefixOnly" has been requested + String queryWildcard = prefixOnly ? String.format("%s%%", description.toLowerCase()) : String.format("%%%s%%", description.toLowerCase()); + sql.append(" AND LCASE(description) LIKE ?"); + bindParams.add(queryWildcard); + } + // Handle name searches if (names != null && !names.isEmpty()) { sql.append(" AND ("); @@ -763,8 +778,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { Long updated = resultSet.getLong(7); // Optional metadata fields - String title = resultSet.getString(8); - String description = resultSet.getString(9); + String titleResult = resultSet.getString(8); + String descriptionResult = resultSet.getString(9); String category = resultSet.getString(10); String tag1 = resultSet.getString(11); String tag2 = resultSet.getString(12); @@ -792,8 +807,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { if (includeMetadata != null && includeMetadata) { // TODO: we could avoid the join altogether ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); - metadata.setTitle(title); - metadata.setDescription(description); + metadata.setTitle(titleResult); + metadata.setDescription(descriptionResult); metadata.setCategory(Category.uncategorizedValueOf(category)); List tags = new ArrayList<>(); diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js index 86493b48..6c72a410 100644 --- a/src/main/resources/q-apps/q-apps.js +++ b/src/main/resources/q-apps/q-apps.js @@ -218,6 +218,8 @@ window.addEventListener("message", (event) => { if (data.identifier != null) url = url.concat("&identifier=" + data.identifier); if (data.name != null) url = url.concat("&name=" + data.name); if (data.names != null) data.names.forEach((x, i) => url = url.concat("&name=" + x)); + if (data.title != null) url = url.concat("&title=" + data.title); + if (data.description != null) url = url.concat("&description=" + data.description); if (data.prefix != null) url = url.concat("&prefix=" + new Boolean(data.prefix).toString()); if (data.exactMatchNames != null) url = url.concat("&exactmatchnames=" + new Boolean(data.exactMatchNames).toString()); if (data.default != null) url = url.concat("&default=" + new Boolean(data.default).toString()); From 8fa344125c6a0364324c271cc335f8ad3218bb22 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 8 May 2023 09:49:10 +0100 Subject: [PATCH 11/34] Fixed issue updating cache when receiving metadata via the network. --- .../controller/arbitrary/ArbitraryMetadataManager.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java index f99ec953..02cf12c9 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java @@ -321,7 +321,7 @@ public class ArbitraryMetadataManager { return; } - // Update requests map to reflect that we've received all chunks + // Update requests map to reflect that we've received this metadata Triple newEntry = new Triple<>(null, null, request.getC()); arbitraryMetadataRequests.put(message.getId(), newEntry); @@ -356,6 +356,7 @@ public class ArbitraryMetadataManager { // Update arbitrary resource caches if (arbitraryTransactionData != null) { + repository.discardChanges(); ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, arbitraryTransactionData); arbitraryTransaction.updateArbitraryResourceCache(); arbitraryTransaction.updateArbitraryMetadataCache(); @@ -363,7 +364,7 @@ public class ArbitraryMetadataManager { } } catch (DataException e) { - LOGGER.error(String.format("Repository issue while finding arbitrary transaction metadata for peer %s", peer), e); + LOGGER.error(String.format("Repository issue while saving arbitrary transaction metadata from peer %s", peer), e); } } From 0ec661431c40221b0aaa3baf6837a3e86087fb7a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 8 May 2023 12:46:15 +0100 Subject: [PATCH 12/34] Added optional "before" and "after" params to `GET /arbitrary/resources/search` --- Q-Apps.md | 2 ++ .../org/qortal/api/resource/ArbitraryResource.java | 7 +++++-- .../org/qortal/repository/ArbitraryRepository.java | 2 +- .../repository/hsqldb/HSQLDBArbitraryRepository.java | 12 +++++++++++- src/main/resources/q-apps/q-apps.js | 2 ++ 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Q-Apps.md b/Q-Apps.md index 177fee2d..41d5d756 100644 --- a/Q-Apps.md +++ b/Q-Apps.md @@ -367,6 +367,8 @@ let res = await qortalRequest({ nameListFilter: "QApp1234Subscriptions", // Optional - will only return results if they are from a name included in supplied list followedOnly: false, // Optional - include followed names only excludeBlocked: false, // Optional - exclude blocked content + // before: 1683546000000, // Optional - limit to resources created before timestamp + // after: 1683546000000, // Optional - limit to resources created after timestamp limit: 100, offset: 0, reverse: true diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 8762a7f1..b3fd6d6d 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -175,6 +175,8 @@ public class ArbitraryResource { @Parameter(description = "Exclude blocked content") @QueryParam("excludeblocked") Boolean excludeBlocked, @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus, @Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata, + @Parameter(description = "Creation date before timestamp") @QueryParam("before") Long before, + @Parameter(description = "Creation date after timestamp") @QueryParam("after") Long after, @Parameter(ref = "limit") @QueryParam("limit") Integer limit, @Parameter(ref = "offset") @QueryParam("offset") Integer offset, @Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) { @@ -203,8 +205,9 @@ public class ArbitraryResource { } List resources = repository.getArbitraryRepository() - .searchArbitraryResources(service, query, identifier, names, title, description, usePrefixOnly, exactMatchNames, - defaultRes, followedOnly, excludeBlocked, includeMetadata, includeStatus, limit, offset, reverse); + .searchArbitraryResources(service, query, identifier, names, title, description, usePrefixOnly, + exactMatchNames, defaultRes, followedOnly, excludeBlocked, includeMetadata, includeStatus, + before, after, limit, offset, reverse); if (resources == null) { return new ArrayList<>(); diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 590aa3b8..089ca199 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -39,7 +39,7 @@ public interface ArbitraryRepository { public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; - public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, List namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; + public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, List namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException; // Arbitrary resources cache save/load diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 45706e11..d72d233b 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -640,7 +640,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { @Override public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, List exactMatchNames, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, - Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException { + Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); @@ -724,6 +724,16 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { sql.append(")"); } + // Timestamp range + if (before != null) { + sql.append(" AND created_when < ?"); + bindParams.add(before); + } + if (after != null) { + sql.append(" AND created_when > ?"); + bindParams.add(after); + } + // Handle "followed only" if (followedOnly != null && followedOnly) { List followedNames = ListUtils.followedNames(); diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js index 6c72a410..ac0d6603 100644 --- a/src/main/resources/q-apps/q-apps.js +++ b/src/main/resources/q-apps/q-apps.js @@ -228,6 +228,8 @@ window.addEventListener("message", (event) => { if (data.nameListFilter != null) url = url.concat("&namefilter=" + data.nameListFilter); if (data.followedOnly != null) url = url.concat("&followedonly=" + new Boolean(data.followedOnly).toString()); if (data.excludeBlocked != null) url = url.concat("&excludeblocked=" + new Boolean(data.excludeBlocked).toString()); + if (data.before != null) url = url.concat("&before=" + data.before); + if (data.after != null) url = url.concat("&after=" + data.after); if (data.limit != null) url = url.concat("&limit=" + data.limit); if (data.offset != null) url = url.concat("&offset=" + data.offset); if (data.reverse != null) url = url.concat("&reverse=" + new Boolean(data.reverse).toString()); From c210d63c40f2ecc882281004407325558567359d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 8 May 2023 13:41:23 +0100 Subject: [PATCH 13/34] Added "mode" parameter to `GET /arbitrary/resources/search`, with possible values of LATEST, ALL. By default, only the latest resource is returned for a name/service combination. All identifiers can be optionally returned by setting `mode` to "ALL". More search modes can be added in the future, for instance "RELEVANT" or "POPULAR" (these are just ideas, and are not currently supported). --- Q-Apps.md | 5 ++++ src/main/java/org/qortal/api/SearchMode.java | 6 +++++ .../api/resource/ArbitraryResource.java | 3 ++- .../repository/ArbitraryRepository.java | 3 ++- .../hsqldb/HSQLDBArbitraryRepository.java | 27 ++++++++++++++++--- src/main/resources/q-apps/q-apps.js | 1 + 6 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/qortal/api/SearchMode.java diff --git a/Q-Apps.md b/Q-Apps.md index 41d5d756..74b09791 100644 --- a/Q-Apps.md +++ b/Q-Apps.md @@ -362,6 +362,7 @@ let res = await qortalRequest({ prefix: false, // Optional - if true, only the beginning of fields are matched in all of the above filters exactMatchNames: true, // Optional - if true, partial name matches are excluded default: false, // Optional - if true, only resources without identifiers are returned + mode: "LATEST", // Optional - whether to return all resources or just the latest for a name/service combination. Possible values: ALL,LATEST. Default: LATEST includeStatus: false, // Optional - will take time to respond, so only request if necessary includeMetadata: false, // Optional - will take time to respond, so only request if necessary nameListFilter: "QApp1234Subscriptions", // Optional - will only return results if they are from a name included in supplied list @@ -384,12 +385,16 @@ let res = await qortalRequest({ identifier: "search query goes here", // Optional - searches only the "identifier" field names: ["QortalDemo", "crowetic", "AlphaX"], // Optional - searches only the "name" field for any of the supplied names prefix: false, // Optional - if true, only the beginning of fields are matched in all of the above filters + exactMatchNames: true, // Optional - if true, partial name matches are excluded default: false, // Optional - if true, only resources without identifiers are returned + mode: "LATEST", // Optional - whether to return all resources or just the latest for a name/service combination. Possible values: ALL,LATEST. Default: LATEST includeStatus: false, // Optional - will take time to respond, so only request if necessary includeMetadata: false, // Optional - will take time to respond, so only request if necessary nameListFilter: "QApp1234Subscriptions", // Optional - will only return results if they are from a name included in supplied list followedOnly: false, // Optional - include followed names only excludeBlocked: false, // Optional - exclude blocked content + // before: 1683546000000, // Optional - limit to resources created before timestamp + // after: 1683546000000, // Optional - limit to resources created after timestamp limit: 100, offset: 0, reverse: true diff --git a/src/main/java/org/qortal/api/SearchMode.java b/src/main/java/org/qortal/api/SearchMode.java new file mode 100644 index 00000000..85c1c61a --- /dev/null +++ b/src/main/java/org/qortal/api/SearchMode.java @@ -0,0 +1,6 @@ +package org.qortal.api; + +public enum SearchMode { + LATEST, + ALL; +} diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index b3fd6d6d..295fb7c5 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -170,6 +170,7 @@ public class ArbitraryResource { @Parameter(description = "Prefix only (if true, only the beginning of fields are matched)") @QueryParam("prefix") Boolean prefixOnly, @Parameter(description = "Exact match names only (if true, partial name matches are excluded)") @QueryParam("exactmatchnames") Boolean exactMatchNamesOnly, @Parameter(description = "Default resources (without identifiers) only") @QueryParam("default") Boolean defaultResource, + @Parameter(description = "Search mode") @QueryParam("mode") SearchMode mode, @Parameter(description = "Filter names by list (exact matches only)") @QueryParam("namefilter") String nameListFilter, @Parameter(description = "Include followed names only") @QueryParam("followedonly") Boolean followedOnly, @Parameter(description = "Exclude blocked content") @QueryParam("excludeblocked") Boolean excludeBlocked, @@ -206,7 +207,7 @@ public class ArbitraryResource { List resources = repository.getArbitraryRepository() .searchArbitraryResources(service, query, identifier, names, title, description, usePrefixOnly, - exactMatchNames, defaultRes, followedOnly, excludeBlocked, includeMetadata, includeStatus, + exactMatchNames, defaultRes, mode, followedOnly, excludeBlocked, includeMetadata, includeStatus, before, after, limit, offset, reverse); if (resources == null) { diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 089ca199..e773597d 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -1,5 +1,6 @@ package org.qortal.repository; +import org.qortal.api.SearchMode; import org.qortal.arbitrary.misc.Service; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceMetadata; @@ -39,7 +40,7 @@ public interface ArbitraryRepository { public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; - public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, List namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException; + public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, List namesFilter, boolean defaultResource, SearchMode mode, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException; // Arbitrary resources cache save/load diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index d72d233b..b1e878ac 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -2,6 +2,7 @@ package org.qortal.repository.hsqldb; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.api.SearchMode; import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Service; @@ -639,16 +640,34 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { @Override public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, - List exactMatchNames, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, + List exactMatchNames, boolean defaultResource, SearchMode mode, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + - "FROM ArbitraryResourcesCache " + - "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + - "WHERE name IS NOT NULL"); + "FROM ArbitraryResourcesCache"); + + // Default to "latest" mode + if (mode == null) { + mode = SearchMode.LATEST; + } + + switch (mode) { + case LATEST: + // Include latest item only for a name/service combination + sql.append(" JOIN (SELECT name, service, MAX(created_when) AS latest " + + "FROM ArbitraryResourcesCache GROUP BY name, service) LatestResources " + + "ON name=LatestResources.name AND service=LatestResources.service " + + "AND created_when=LatestResources.latest"); + break; + + case ALL: + break; + } + + sql.append(" LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) WHERE name IS NOT NULL"); if (service != null) { sql.append(" AND service = "); diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js index ac0d6603..3069c160 100644 --- a/src/main/resources/q-apps/q-apps.js +++ b/src/main/resources/q-apps/q-apps.js @@ -223,6 +223,7 @@ window.addEventListener("message", (event) => { if (data.prefix != null) url = url.concat("&prefix=" + new Boolean(data.prefix).toString()); if (data.exactMatchNames != null) url = url.concat("&exactmatchnames=" + new Boolean(data.exactMatchNames).toString()); if (data.default != null) url = url.concat("&default=" + new Boolean(data.default).toString()); + if (data.mode != null) url = url.concat("&mode=" + data.mode); if (data.includeStatus != null) url = url.concat("&includestatus=" + new Boolean(data.includeStatus).toString()); if (data.includeMetadata != null) url = url.concat("&includemetadata=" + new Boolean(data.includeMetadata).toString()); if (data.nameListFilter != null) url = url.concat("&namefilter=" + data.nameListFilter); From 7725c5e21fe5fdad8175d468883cce00ca033528 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 12 May 2023 12:03:32 +0100 Subject: [PATCH 14/34] Always ignore unsupported services when building the cache. --- .../org/qortal/transaction/ArbitraryTransaction.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index b2954dd9..df4a92f7 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -369,6 +369,11 @@ public class ArbitraryTransaction extends Transaction { String name = arbitraryTransactionData.getName(); String identifier = arbitraryTransactionData.getIdentifier(); + if (service == null) { + // Unsupported service - ignore this resource + return; + } + // In the cache we store null identifiers as "default", as it is part of the primary key if (identifier == null) { identifier = "default"; @@ -427,6 +432,11 @@ public class ArbitraryTransaction extends Transaction { String name = latestTransactionData.getName(); String identifier = latestTransactionData.getIdentifier(); + if (service == null) { + // Unsupported service - ignore this resource + return; + } + // In the cache we store null identifiers as "default", as it is part of the primary key if (identifier == null) { identifier = "default"; From b661d39844a7d776cc7c6ce848a2e84ec9224fdc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 12 May 2023 19:39:31 +0100 Subject: [PATCH 15/34] Cache updating moved to a dedicated thread. Hopeful fix for serialization failures which occurred when updating from various different network threads. --- .../api/resource/ArbitraryResource.java | 3 +- .../org/qortal/controller/Controller.java | 4 +- .../arbitrary/ArbitraryDataCacheManager.java | 167 ++++++++++++++++++ .../arbitrary/ArbitraryDataManager.java | 6 +- .../arbitrary/ArbitraryMetadataManager.java | 9 +- .../qortal/repository/RepositoryManager.java | 64 ------- .../transaction/ArbitraryTransaction.java | 11 +- 7 files changed, 181 insertions(+), 83 deletions(-) create mode 100644 src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 295fb7c5..7dae7483 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -45,6 +45,7 @@ import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.Controller; +import org.qortal.controller.arbitrary.ArbitraryDataCacheManager; import org.qortal.controller.arbitrary.ArbitraryDataRenderManager; import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; import org.qortal.controller.arbitrary.ArbitraryMetadataManager; @@ -1133,7 +1134,7 @@ public class ArbitraryResource { Security.checkApiCallAllowed(request); try (final Repository repository = RepositoryManager.getRepository()) { - RepositoryManager.buildArbitraryResourcesCache(repository, true); + ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, true); return "true"; } catch (DataException e) { diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 1de6f776..16373099 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -403,7 +403,7 @@ public class Controller extends Thread { RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); try (final Repository repository = RepositoryManager.getRepository()) { - RepositoryManager.buildArbitraryResourcesCache(repository, false); + ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, false); } } catch (DataException e) { @@ -485,6 +485,7 @@ public class Controller extends Thread { LOGGER.info("Starting arbitrary-transaction controllers"); ArbitraryDataManager.getInstance().start(); ArbitraryDataFileManager.getInstance().start(); + ArbitraryDataCacheManager.getInstance().start(); ArbitraryDataBuildManager.getInstance().start(); ArbitraryDataCleanupManager.getInstance().start(); ArbitraryDataStorageManager.getInstance().start(); @@ -939,6 +940,7 @@ public class Controller extends Thread { LOGGER.info("Shutting down arbitrary-transaction controllers"); ArbitraryDataManager.getInstance().shutdown(); ArbitraryDataFileManager.getInstance().shutdown(); + ArbitraryDataCacheManager.getInstance().shutdown(); ArbitraryDataBuildManager.getInstance().shutdown(); ArbitraryDataCleanupManager.getInstance().shutdown(); ArbitraryDataStorageManager.getInstance().shutdown(); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java new file mode 100644 index 00000000..df2c1f29 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java @@ -0,0 +1,167 @@ +package org.qortal.controller.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.api.resource.TransactionsResource; +import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryResourceData; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.gui.SplashFrame; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.transaction.ArbitraryTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.utils.Base58; + +import java.util.*; + +public class ArbitraryDataCacheManager extends Thread { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCacheManager.class); + + private static ArbitraryDataCacheManager instance; + private volatile boolean isStopping = false; + + /** Queue of arbitrary transactions that require cache updates */ + private final List updateQueue = Collections.synchronizedList(new ArrayList<>()); + + + public static synchronized ArbitraryDataCacheManager getInstance() { + if (instance == null) { + instance = new ArbitraryDataCacheManager(); + } + + return instance; + } + + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data Cache Manager"); + + try { + while (!Controller.isStopping()) { + Thread.sleep(500L); + + // Process queue + processResourceQueue(); + } + } catch (InterruptedException e) { + // Fall through to exit thread + } + } + + public void shutdown() { + isStopping = true; + this.interrupt(); + } + + + private void processResourceQueue() { + if (this.updateQueue.isEmpty()) { + // Nothing to do + return; + } + + try (final Repository repository = RepositoryManager.getRepository()) { + // Take a snapshot of resourceQueue, so we don't need to lock it while processing + List resourceQueueCopy = List.copyOf(this.updateQueue); + + for (ArbitraryTransactionData transactionData : resourceQueueCopy) { + // Best not to return when controller is stopping, as ideally we need to finish processing + + LOGGER.debug(() -> String.format("Processing transaction %.8s in arbitrary resource queue...", Base58.encode(transactionData.getSignature()))); + + // Remove from the queue regardless of outcome + this.updateQueue.remove(transactionData); + + // Update arbitrary resource caches + try { + ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); + arbitraryTransaction.updateArbitraryResourceCache(); + arbitraryTransaction.updateArbitraryMetadataCache(); + repository.saveChanges(); + + LOGGER.debug(() -> String.format("Finished processing transaction %.8s in arbitrary resource queue...", Base58.encode(transactionData.getSignature()))); + + } catch (DataException e) { + repository.discardChanges(); + LOGGER.error("Repository issue while updating arbitrary resource caches", e); + } + } + } catch (DataException e) { + LOGGER.error("Repository issue while processing arbitrary resource cache updates", e); + } + } + + public void addToUpdateQueue(ArbitraryTransactionData transactionData) { + this.updateQueue.add(transactionData); + LOGGER.debug(() -> String.format("Transaction %.8s added to queue", Base58.encode(transactionData.getSignature()))); + } + + public boolean buildArbitraryResourcesCache(Repository repository, boolean forceRebuild) throws DataException { + if (Settings.getInstance().isLite()) { + // Lite nodes have no blockchain + return false; + } + + try { + // Check if QDNResources table is empty + List resources = repository.getArbitraryRepository().getArbitraryResources(10, 0, false); + if (!resources.isEmpty() && !forceRebuild) { + // Resources exist in the cache, so assume complete. + // We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so + // we shouldn't ever be left in a partially rebuilt state. + LOGGER.debug("Arbitrary resources cache already built"); + return false; + } + + LOGGER.info("Building arbitrary resources cache..."); + SplashFrame.getInstance().updateStatus("Building QDN cache - please wait..."); + + final int batchSize = 100; + int offset = 0; + + // Loop through all ARBITRARY transactions, and determine latest state + while (!Controller.isStopping()) { + LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1); + + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false); + if (signatures.isEmpty()) { + // Complete + break; + } + + // Expand signatures to transactions + for (byte[] signature : signatures) { + ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository + .getTransactionRepository().fromSignature(signature); + + if (transactionData.getService() == null) { + // Unsupported service - ignore this resource + continue; + } + + // Update arbitrary resource caches + ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); + arbitraryTransaction.updateArbitraryResourceCache(); + arbitraryTransaction.updateArbitraryMetadataCache(); + } + offset += batchSize; + } + + repository.saveChanges(); + LOGGER.info("Completed build of arbitrary resources cache."); + return true; + } + catch (DataException e) { + LOGGER.info("Unable to build arbitrary resources cache: {}. The database may have been left in an inconsistent state.", e.getMessage()); + + // Throw an exception so that the node startup is halted, allowing for a retry next time. + repository.discardChanges(); + throw new DataException("Build of arbitrary resources cache failed."); + } + } + +} diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 60bf63b3..f70354d6 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -564,10 +564,8 @@ public class ArbitraryDataManager extends Thread { repository.getArbitraryRepository().delete(arbitraryResourceData); } else { - // We found the next oldest transaction, so we can update the cache - ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, latestTransactionData); - arbitraryTransaction.updateArbitraryResourceCache(); - arbitraryTransaction.updateArbitraryMetadataCache(); + // We found the next oldest transaction, so add to queue for processing + ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData); } ; repository.saveChanges(); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java index 02cf12c9..1fee6753 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java @@ -15,7 +15,6 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; -import org.qortal.transaction.ArbitraryTransaction; import org.qortal.utils.Base58; import org.qortal.utils.ListUtils; import org.qortal.utils.NTP; @@ -354,13 +353,9 @@ public class ArbitraryMetadataManager { } } - // Update arbitrary resource caches + // Add to resource queue to update arbitrary resource caches if (arbitraryTransactionData != null) { - repository.discardChanges(); - ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, arbitraryTransactionData); - arbitraryTransaction.updateArbitraryResourceCache(); - arbitraryTransaction.updateArbitraryMetadataCache(); - repository.saveChanges(); + ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData); } } catch (DataException e) { diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index 26eb49e3..fcf9e398 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -65,70 +65,6 @@ public abstract class RepositoryManager { } } - public static boolean buildArbitraryResourcesCache(Repository repository, boolean forceRebuild) throws DataException { - if (Settings.getInstance().isLite()) { - // Lite nodes have no blockchain - return false; - } - - try { - // Check if QDNResources table is empty - List resources = repository.getArbitraryRepository().getArbitraryResources(10, 0, false); - if (!resources.isEmpty() && !forceRebuild) { - // Resources exist in the cache, so assume complete. - // We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so - // we shouldn't ever be left in a partially rebuilt state. - LOGGER.debug("Arbitrary resources cache already built"); - return false; - } - - LOGGER.info("Building arbitrary resources cache..."); - SplashFrame.getInstance().updateStatus("Building QDN cache - please wait..."); - - final int batchSize = 100; - int offset = 0; - - // Loop through all ARBITRARY transactions, and determine latest state - while (!Controller.isStopping()) { - LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1); - - List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false); - if (signatures.isEmpty()) { - // Complete - break; - } - - // Expand signatures to transactions - for (byte[] signature : signatures) { - ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository - .getTransactionRepository().fromSignature(signature); - - if (transactionData.getService() == null) { - // Unsupported service - ignore this resource - continue; - } - - // Update arbitrary resource caches - ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); - arbitraryTransaction.updateArbitraryResourceCache(); - arbitraryTransaction.updateArbitraryMetadataCache(); - } - offset += batchSize; - } - - repository.saveChanges(); - LOGGER.info("Completed build of arbitrary resources cache."); - return true; - } - catch (DataException e) { - LOGGER.info("Unable to build arbitrary resources cache: {}. The database may have been left in an inconsistent state.", e.getMessage()); - - // Throw an exception so that the node startup is halted, allowing for a retry next time. - repository.discardChanges(); - throw new DataException("Build of arbitrary resources cache failed."); - } - } - public static void setRequestedCheckpoint(Boolean quick) { quickCheckpointRequested = quick; } diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index df4a92f7..d931e231 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -12,6 +12,7 @@ import org.qortal.arbitrary.ArbitraryDataResource; import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Service; import org.qortal.block.BlockChain; +import org.qortal.controller.arbitrary.ArbitraryDataCacheManager; import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.crypto.Crypto; @@ -258,9 +259,8 @@ public class ArbitraryTransaction extends Transaction { } } - // Add to arbitrary resource caches - this.updateArbitraryResourceCache(); - this.updateArbitraryMetadataCache(); + // Add to queue for cache updates + ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData); } @Override @@ -296,9 +296,8 @@ public class ArbitraryTransaction extends Transaction { } } - // Add/update arbitrary resource caches - this.updateArbitraryResourceCache(); - this.updateArbitraryMetadataCache(); + // Add to queue for cache updates + ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData); } catch (Exception e) { // Log and ignore all exceptions. The cache is updated from other places too, and can be rebuilt if needed. From 36a731255ad311499fbfb3b2964a4a566639115e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 12 May 2023 20:08:53 +0100 Subject: [PATCH 16/34] Automatically delete cached resources & metadata if there is no longer a latest transaction. --- .../arbitrary/ArbitraryDataManager.java | 27 ++----------------- .../transaction/ArbitraryTransaction.java | 23 ++++++++-------- 2 files changed, 14 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index f70354d6..1e24ae70 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -546,32 +546,9 @@ public class ArbitraryDataManager extends Thread { return; } - Service service = arbitraryTransactionData.getService(); - String name = arbitraryTransactionData.getName(); - String identifier = arbitraryTransactionData.getIdentifier(); + // Add to queue for update/deletion + ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData); - ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); - arbitraryResourceData.service = service; - arbitraryResourceData.name = name; - arbitraryResourceData.identifier = identifier; - - try (final Repository repository = RepositoryManager.getRepository()) { - // Find next oldest transaction (which is now the latest transaction) - ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(name, service, null, identifier); - - if (latestTransactionData == null) { - // There are no transactions anymore, so we can delete from the cache entirely (this deletes metadata too) - repository.getArbitraryRepository().delete(arbitraryResourceData); - } - else { - // We found the next oldest transaction, so add to queue for processing - ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData); - } -; - repository.saveChanges(); - } catch (DataException e) { - // Not much we can do, so ignore for now - } } public int getPowDifficulty() { diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index d931e231..46b26f6b 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -357,13 +357,6 @@ public class ArbitraryTransaction extends Transaction { return; } - // Get the latest transaction - ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); - if (latestTransactionData == null) { - // We don't have a latest transaction, so give up - return; - } - Service service = arbitraryTransactionData.getService(); String name = arbitraryTransactionData.getName(); String identifier = arbitraryTransactionData.getIdentifier(); @@ -378,15 +371,23 @@ public class ArbitraryTransaction extends Transaction { identifier = "default"; } - // Get existing cached entry if it exists - ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository() - .getArbitraryResource(service, name, identifier); - ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); arbitraryResourceData.service = service; arbitraryResourceData.name = name; arbitraryResourceData.identifier = identifier; + // Get the latest transaction + ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); + if (latestTransactionData == null) { + // We don't have a latest transaction, so delete from cache + repository.getArbitraryRepository().delete(arbitraryResourceData); + return; + } + + // Get existing cached entry if it exists + ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository() + .getArbitraryResource(service, name, identifier); + // Check for existing cached data if (existingArbitraryResourceData == null) { // Nothing exists yet, so set everything from the newest transaction From 23d211836fa9d0123f03b5ef26ed039f779dc31f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 12 May 2023 20:10:51 +0100 Subject: [PATCH 17/34] Fixed case sensitivity issue when updating status in the cache. --- .../qortal/repository/hsqldb/HSQLDBArbitraryRepository.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index b1e878ac..e336d140 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -886,10 +886,10 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { if (status == null) { return; } - String updateSql = "UPDATE ArbitraryResourcesCache SET status = ? WHERE service = ? AND name = ? AND identifier = ?"; + String updateSql = "UPDATE ArbitraryResourcesCache SET status = ? WHERE service = ? AND LCASE(name) = ? AND LCASE(identifier) = ?"; try { - this.repository.executeCheckedUpdate(updateSql, status.value, arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier); + this.repository.executeCheckedUpdate(updateSql, status.value, arbitraryResourceData.service.value, arbitraryResourceData.name.toLowerCase(), arbitraryResourceData.identifier.toLowerCase()); } catch (SQLException e) { throw new DataException("Unable to set status for arbitrary resource", e); } From 5c7d12f25e628e094feb5fcc90652ab702143573 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 13 May 2023 12:27:56 +0100 Subject: [PATCH 18/34] Fixed bug causing incorrect creation dates in the cache. --- .../java/org/qortal/transaction/ArbitraryTransaction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 46b26f6b..ab5adc76 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -390,8 +390,8 @@ public class ArbitraryTransaction extends Transaction { // Check for existing cached data if (existingArbitraryResourceData == null) { - // Nothing exists yet, so set everything from the newest transaction - arbitraryResourceData.created = latestTransactionData.getTimestamp(); + // Nothing exists yet, so set creation date from the current transaction (it will be reduced later if needed) + arbitraryResourceData.created = arbitraryTransactionData.getTimestamp(); arbitraryResourceData.updated = null; } else { From 5ed3237d2f73c4bdfd276a82dd755f3bfb0cd63e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 13 May 2023 13:36:40 +0100 Subject: [PATCH 19/34] Clear queue before exiting cache manager thread. --- .../qortal/controller/arbitrary/ArbitraryDataCacheManager.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java index df2c1f29..f12767b9 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java @@ -50,6 +50,9 @@ public class ArbitraryDataCacheManager extends Thread { } catch (InterruptedException e) { // Fall through to exit thread } + + // Clear queue before terminating thread + processResourceQueue(); } public void shutdown() { From f451bccbf66315ad027cc50d6a8d6985ae9f7e65 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 13 May 2023 14:54:00 +0100 Subject: [PATCH 20/34] Fixed bug causing descriptions to be truncated in the cache. --- .../org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index e336d140..4be2e7b4 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -922,7 +922,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { // Trim metadata values if they are too long to fit in the db String title = ArbitraryDataTransactionMetadata.limitTitle(metadata.getTitle()); - String description = ArbitraryDataTransactionMetadata.limitTitle(metadata.getDescription()); + String description = ArbitraryDataTransactionMetadata.limitDescription(metadata.getDescription()); List tags = ArbitraryDataTransactionMetadata.limitTags(metadata.getTags()); String tag1 = null; From a49529ad9b52efe5ea6df9834eb143cfd0d0a9c5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 13 May 2023 15:11:32 +0100 Subject: [PATCH 21/34] Cache updating moved back to existing threads when processing or importing a transaction, to remove chances of queued updates being lost. The dedicated cache manager thread is now used for metadata updates only. If metadata ever goes missing from the db, it would be straightforward to have a background thread that corrects any discrepancies between the filesystem and the db. Not adding that until it is needed. --- .../transaction/ArbitraryTransaction.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index ab5adc76..a189adf2 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -250,17 +250,8 @@ public class ArbitraryTransaction extends Transaction { // We may need to move files from the misc_ folder ArbitraryTransactionUtils.checkAndRelocateMiscFiles(arbitraryTransactionData); - // If the data is local, we need to perform a few actions - if (isDataLocal()) { - - // We have the data for this transaction, so invalidate the cache - if (arbitraryTransactionData.getName() != null) { - ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData); - } - } - - // Add to queue for cache updates - ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData); + // Update caches + updateCaches(); } @Override @@ -296,8 +287,9 @@ public class ArbitraryTransaction extends Transaction { } } - // Add to queue for cache updates - ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData); + // Add/update arbitrary resource caches + this.updateArbitraryResourceCache(); + this.updateArbitraryMetadataCache(); } catch (Exception e) { // Log and ignore all exceptions. The cache is updated from other places too, and can be rebuilt if needed. From 633f73aa86dd34321f9ea599208264552e11bf05 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 13 May 2023 16:32:33 +0100 Subject: [PATCH 22/34] Removed API key fields from documentation for methods that don't require an API key by default. --- .../api/resource/ArbitraryResource.java | 33 +++++++------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 7dae7483..b17e810c 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -233,14 +233,12 @@ public class ArbitraryResource { ) } ) - @SecurityRequirement(name = "apiKey") - public ArbitraryResourceStatus getDefaultResourceStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey, - @PathParam("service") Service service, + public ArbitraryResourceStatus getDefaultResourceStatus(@PathParam("service") Service service, @PathParam("name") String name, @QueryParam("build") Boolean build) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) - Security.requirePriorAuthorizationOrApiKey(request, name, service, null, apiKey); + Security.requirePriorAuthorizationOrApiKey(request, name, service, null, null); return ArbitraryTransactionUtils.getStatus(service, name, null, build, true); } @@ -256,14 +254,12 @@ public class ArbitraryResource { ) } ) - @SecurityRequirement(name = "apiKey") - public FileProperties getResourceProperties(@HeaderParam(Security.API_KEY_HEADER) String apiKey, - @PathParam("service") Service service, - @PathParam("name") String name, - @PathParam("identifier") String identifier) { + public FileProperties getResourceProperties(@PathParam("service") Service service, + @PathParam("name") String name, + @PathParam("identifier") String identifier) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) - Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, apiKey); + Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, null); return this.getFileProperties(service, name, identifier); } @@ -279,15 +275,13 @@ public class ArbitraryResource { ) } ) - @SecurityRequirement(name = "apiKey") - public ArbitraryResourceStatus getResourceStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey, - @PathParam("service") Service service, + public ArbitraryResourceStatus getResourceStatus(@PathParam("service") Service service, @PathParam("name") String name, @PathParam("identifier") String identifier, @QueryParam("build") Boolean build) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) - Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, apiKey); + Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, null); return ArbitraryTransactionUtils.getStatus(service, name, identifier, build, true); } @@ -630,9 +624,7 @@ public class ArbitraryResource { ) } ) - @SecurityRequirement(name = "apiKey") - public HttpServletResponse get(@HeaderParam(Security.API_KEY_HEADER) String apiKey, - @PathParam("service") Service service, + public HttpServletResponse get(@PathParam("service") Service service, @PathParam("name") String name, @QueryParam("filepath") String filepath, @QueryParam("encoding") String encoding, @@ -665,9 +657,7 @@ public class ArbitraryResource { ) } ) - @SecurityRequirement(name = "apiKey") - public HttpServletResponse get(@HeaderParam(Security.API_KEY_HEADER) String apiKey, - @PathParam("service") Service service, + public HttpServletResponse get(@PathParam("service") Service service, @PathParam("name") String name, @PathParam("identifier") String identifier, @QueryParam("filepath") String filepath, @@ -678,7 +668,7 @@ public class ArbitraryResource { // Authentication can be bypassed in the settings, for those running public QDN nodes if (!Settings.getInstance().isQDNAuthBypassEnabled()) { - Security.checkApiCallAllowed(request, apiKey); + Security.checkApiCallAllowed(request, null); } return this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts); @@ -703,7 +693,6 @@ public class ArbitraryResource { ) } ) - @SecurityRequirement(name = "apiKey") public ArbitraryResourceMetadata getMetadata(@PathParam("service") Service service, @PathParam("name") String name, @PathParam("identifier") String identifier) { From f5f82dc3f6e60c668fec3ce0ba6622aaec53cdf7 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 13 May 2023 19:20:18 +0100 Subject: [PATCH 23/34] Fixed issues relating to using a separate repository instance when determining the latest status of a resource. --- .../api/gateway/resource/GatewayResource.java | 38 +++------- .../api/resource/ArbitraryResource.java | 10 ++- .../arbitrary/ArbitraryDataResource.java | 75 +++++++++++-------- .../transaction/ArbitraryTransaction.java | 2 +- 4 files changed, 66 insertions(+), 59 deletions(-) diff --git a/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java b/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java index 7d76be3d..15bb398e 100644 --- a/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java +++ b/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java @@ -3,6 +3,8 @@ package org.qortal.api.gateway.resource; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; import org.apache.commons.lang3.StringUtils; +import org.qortal.api.ApiError; +import org.qortal.api.ApiExceptionFactory; import org.qortal.api.Security; import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; @@ -11,6 +13,9 @@ import org.qortal.arbitrary.ArbitraryDataRenderer; import org.qortal.arbitrary.ArbitraryDataResource; import org.qortal.arbitrary.misc.Service; import org.qortal.data.arbitrary.ArbitraryResourceStatus; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; @@ -31,30 +36,6 @@ public class GatewayResource { @Context HttpServletResponse response; @Context ServletContext context; - /** - * We need to allow resource status checking (and building) via the gateway, as the node's API port - * may not be forwarded and will almost certainly not be authenticated. Since gateways allow for - * all resources to be loaded except those that are blocked, there is no need for authentication. - */ - @GET - @Path("/arbitrary/resource/status/{service}/{name}") - public ArbitraryResourceStatus getDefaultResourceStatus(@PathParam("service") Service service, - @PathParam("name") String name, - @QueryParam("build") Boolean build) { - - return this.getStatus(service, name, null, build); - } - - @GET - @Path("/arbitrary/resource/status/{service}/{name}/{identifier}") - public ArbitraryResourceStatus getResourceStatus(@PathParam("service") Service service, - @PathParam("name") String name, - @PathParam("identifier") String identifier, - @QueryParam("build") Boolean build) { - - return this.getStatus(service, name, identifier, build); - } - private ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build) { // If "build=true" has been specified in the query string, build the resource before returning its status @@ -69,8 +50,13 @@ public class GatewayResource { } } - ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); - return resource.getStatus(); + try (final Repository repository = RepositoryManager.getRepository()) { + ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); + return resource.getStatus(repository); + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } } diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index b17e810c..b7eb6cd4 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -531,8 +531,14 @@ public class ArbitraryResource { @PathParam("identifier") String identifier) { Security.checkApiCallAllowed(request); - ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); - return resource.delete(false); + + try (final Repository repository = RepositoryManager.getRepository()) { + ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); + return resource.delete(repository, false); + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } } @POST diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index 4aed21fc..96fc4e48 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -59,28 +59,34 @@ public class ArbitraryDataResource { } public ArbitraryResourceStatus getStatusAndUpdateCache(boolean updateCache) { - ArbitraryResourceStatus arbitraryResourceStatus = this.getStatus(); + ArbitraryResourceStatus arbitraryResourceStatus = null; - if (updateCache) { - // Update cache if possible - ArbitraryResourceStatus.Status status = arbitraryResourceStatus != null ? arbitraryResourceStatus.getStatus() : null; - ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(this.service, this.resourceId, this.identifier); + try (final Repository repository = RepositoryManager.getRepository()) { + arbitraryResourceStatus = this.getStatus(repository); - try (final Repository repository = RepositoryManager.getRepository()) { + if (updateCache) { + // Update cache if possible + ArbitraryResourceStatus.Status status = arbitraryResourceStatus != null ? arbitraryResourceStatus.getStatus() : null; + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(this.service, this.resourceId, this.identifier); repository.getArbitraryRepository().setStatus(arbitraryResourceData, status); repository.saveChanges(); - - } catch (DataException e) { - LOGGER.info("Unable to update status cache for resource {}: {}", arbitraryResourceData, e.getMessage()); } + } catch (DataException e) { + LOGGER.info("Unable to update status cache for resource {}: {}", this.toString(), e.getMessage()); } return arbitraryResourceStatus; } - public ArbitraryResourceStatus getStatus() { + /** + * Get current status of resource + * + * @param repository + * @return the resource's status + */ + public ArbitraryResourceStatus getStatus(Repository repository) { // Calculate the chunk counts - this.calculateChunkCounts(); + this.calculateChunkCounts(repository); if (!this.exists) { return new ArbitraryResourceStatus(Status.NOT_PUBLISHED, this.localChunkCount, this.totalChunkCount); @@ -111,11 +117,11 @@ public class ArbitraryDataResource { } // Check if we have all data locally for this resource - if (!this.allFilesDownloaded()) { - if (this.isDownloading()) { + if (!this.allFilesDownloaded(repository)) { + if (this.isDownloading(repository)) { return new ArbitraryResourceStatus(Status.DOWNLOADING, this.localChunkCount, this.totalChunkCount); } - else if (this.isDataPotentiallyAvailable()) { + else if (this.isDataPotentiallyAvailable(repository)) { return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount); } return new ArbitraryResourceStatus(Status.MISSING_DATA, this.localChunkCount, this.totalChunkCount); @@ -157,9 +163,9 @@ public class ArbitraryDataResource { return null; } - public boolean delete(boolean deleteMetadata) { + public boolean delete(Repository repository, boolean deleteMetadata) { try { - this.fetchTransactions(); + this.fetchTransactions(repository); if (this.transactions == null) { return false; } @@ -208,7 +214,7 @@ public class ArbitraryDataResource { } } - private boolean allFilesDownloaded() { + private boolean allFilesDownloaded(Repository repository) { // Use chunk counts to speed things up if we can if (this.localChunkCount != null && this.totalChunkCount != null && this.localChunkCount >= this.totalChunkCount) { @@ -216,7 +222,7 @@ public class ArbitraryDataResource { } try { - this.fetchTransactions(); + this.fetchTransactions(repository); if (this.transactions == null) { return false; } @@ -236,9 +242,14 @@ public class ArbitraryDataResource { } } - private void calculateChunkCounts() { + /** + * Calculate chunk counts of a resource + * + * @param repository optional - a new instance will be created if null + */ + private void calculateChunkCounts(Repository repository) { try { - this.fetchTransactions(); + this.fetchTransactions(repository); if (this.transactions == null) { this.exists = false; this.localChunkCount = 0; @@ -263,9 +274,9 @@ public class ArbitraryDataResource { } catch (DataException e) {} } - private boolean isRateLimited() { + private boolean isRateLimited(Repository repository) { try { - this.fetchTransactions(); + this.fetchTransactions(repository); if (this.transactions == null) { return true; } @@ -289,9 +300,9 @@ public class ArbitraryDataResource { * This is only used to give an indication to the user of progress * @return - whether data might be available on the network */ - private boolean isDataPotentiallyAvailable() { + private boolean isDataPotentiallyAvailable(Repository repository) { try { - this.fetchTransactions(); + this.fetchTransactions(repository); if (this.transactions == null) { return false; } @@ -324,9 +335,9 @@ public class ArbitraryDataResource { * This is only used to give an indication to the user of progress * @return - whether we are trying to download the resource */ - private boolean isDownloading() { + private boolean isDownloading(Repository repository) { try { - this.fetchTransactions(); + this.fetchTransactions(repository); if (this.transactions == null) { return false; } @@ -357,15 +368,19 @@ public class ArbitraryDataResource { } - - private void fetchTransactions() throws DataException { + /** + * Fetch relevant arbitrary transactions for resource + * + * @param repository + * @throws DataException + */ + private void fetchTransactions(Repository repository) throws DataException { if (this.transactions != null && !this.transactions.isEmpty()) { // Already fetched return; } - try (final Repository repository = RepositoryManager.getRepository()) { - + try { // Get the most recent PUT ArbitraryTransactionData latestPut = repository.getArbitraryRepository() .getLatestTransaction(this.resourceId, this.service, ArbitraryTransactionData.Method.PUT, this.identifier); diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index a189adf2..c0d441e0 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -407,7 +407,7 @@ public class ArbitraryTransaction extends Transaction { // Update status ArbitraryDataResource resource = new ArbitraryDataResource(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); - ArbitraryResourceStatus arbitraryResourceStatus = resource.getStatus(); + ArbitraryResourceStatus arbitraryResourceStatus = resource.getStatus(repository); ArbitraryResourceStatus.Status status = arbitraryResourceStatus != null ? arbitraryResourceStatus.getStatus() : null; repository.getArbitraryRepository().setStatus(arbitraryResourceData, status); } From 707176a202b88567f5ac294c7bffab90d4359e4c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 27 May 2023 11:30:19 +0200 Subject: [PATCH 24/34] Improved detection of an existing arbitrary resources cache. --- .../org/qortal/controller/Controller.java | 7 ++ .../arbitrary/ArbitraryDataCacheManager.java | 30 +++++-- .../repository/ArbitraryRepository.java | 2 + .../hsqldb/HSQLDBArbitraryRepository.java | 79 +++++++++++++++++++ 4 files changed, 112 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index b9c3cde9..b6efd26f 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -450,6 +450,13 @@ public class Controller extends Thread { Gui.getInstance().fatalError("Database upgrade needed", "Please restart the core to complete the upgrade process."); return; } + if (ArbitraryDataCacheManager.getInstance().needsArbitraryResourcesCacheRebuild(repository)) { + // Don't allow the node to start if arbitrary resources cache hasn't been built yet + // This is needed to handle a case when bootstrapping + LOGGER.error("Database upgrade needed. Please restart the core to complete the upgrade process."); + Gui.getInstance().fatalError("Database upgrade needed", "Please restart the core to complete the upgrade process."); + return; + } } catch (DataException e) { LOGGER.error("Error checking transaction sequences in repository", e); return; diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java index f12767b9..75b00452 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java @@ -103,6 +103,28 @@ public class ArbitraryDataCacheManager extends Thread { LOGGER.debug(() -> String.format("Transaction %.8s added to queue", Base58.encode(transactionData.getSignature()))); } + public boolean needsArbitraryResourcesCacheRebuild(Repository repository) throws DataException { + // Check if we have an entry in the cache for the oldest ARBITRARY transaction with a name + List oldestCacheableTransactions = repository.getArbitraryRepository().getArbitraryTransactions(true, 1, 0, false); + if (oldestCacheableTransactions == null || oldestCacheableTransactions.isEmpty()) { + // No relevant arbitrary transactions yet on this chain + LOGGER.debug("No relevant arbitrary transactions exist to build cache from"); + return false; + } + // We have an arbitrary transaction, so check if it's in the cache + ArbitraryTransactionData txn = oldestCacheableTransactions.get(0); + ArbitraryResourceData cachedResource = repository.getArbitraryRepository().getArbitraryResource(txn.getService(), txn.getName(), txn.getIdentifier()); + if (cachedResource != null) { + // Earliest resource exists in the cache, so assume it has been built. + // We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so + // we shouldn't ever be left in a partially rebuilt state. + LOGGER.debug("Arbitrary resources cache already built"); + return false; + } + + return true; + } + public boolean buildArbitraryResourcesCache(Repository repository, boolean forceRebuild) throws DataException { if (Settings.getInstance().isLite()) { // Lite nodes have no blockchain @@ -110,12 +132,8 @@ public class ArbitraryDataCacheManager extends Thread { } try { - // Check if QDNResources table is empty - List resources = repository.getArbitraryRepository().getArbitraryResources(10, 0, false); - if (!resources.isEmpty() && !forceRebuild) { - // Resources exist in the cache, so assume complete. - // We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so - // we shouldn't ever be left in a partially rebuilt state. + // Skip if already built + if (!needsArbitraryResourcesCacheRebuild(repository) && !forceRebuild) { LOGGER.debug("Arbitrary resources cache already built"); return false; } diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index e773597d..8cff1231 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -31,6 +31,8 @@ public interface ArbitraryRepository { public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException; + public List getArbitraryTransactions(boolean requireName, Integer limit, Integer offset, Boolean reverse) throws DataException; + // Resource related diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 4be2e7b4..a20036de 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -316,6 +316,85 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { return this.getSingleTransaction(name, service, method, identifier, false); } + public List getArbitraryTransactions(boolean requireName, Integer limit, Integer offset, Boolean reverse) throws DataException { + StringBuilder sql = new StringBuilder(512); + sql.append("SELECT type, reference, signature, creator, created_when, fee, " + + "tx_group_id, block_height, approval_status, approval_height, " + + "version, nonce, service, size, is_data_raw, data, metadata_hash, " + + "name, identifier, update_method, secret, compression FROM ArbitraryTransactions " + + "JOIN Transactions USING (signature)"); + + if (requireName) { + sql.append(" WHERE name IS NOT NULL"); + } + + sql.append(" ORDER BY created_when"); + + if (reverse != null && reverse) { + sql.append(" DESC"); + } + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List arbitraryTransactionData = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) { + if (resultSet == null) + return null; + + do { + //TransactionType type = TransactionType.valueOf(resultSet.getInt(1)); + + byte[] reference = resultSet.getBytes(2); + byte[] signature = resultSet.getBytes(3); + byte[] creatorPublicKey = resultSet.getBytes(4); + long timestamp = resultSet.getLong(5); + + Long fee = resultSet.getLong(6); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(7); + + Integer blockHeight = resultSet.getInt(8); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9)); + Integer approvalHeight = resultSet.getInt(10); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + int version = resultSet.getInt(11); + int nonce = resultSet.getInt(12); + int serviceInt = resultSet.getInt(13); + int size = resultSet.getInt(14); + boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false + DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; + byte[] data = resultSet.getBytes(16); + byte[] metadataHash = resultSet.getBytes(17); + String nameResult = resultSet.getString(18); + String identifierResult = resultSet.getString(19); + Method method = Method.valueOf(resultSet.getInt(20)); + byte[] secret = resultSet.getBytes(21); + Compression compression = Compression.valueOf(resultSet.getInt(22)); + // FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls. + + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + version, serviceInt, nonce, size, nameResult, identifierResult, method, secret, + compression, data, dataType, metadataHash, null); + + arbitraryTransactionData.add(transactionData); + } while (resultSet.next()); + + return arbitraryTransactionData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); + } + } + // Resource related From 7e872f78005cc1bf91b976b9886af6e86fd4ed79 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 16 Jun 2023 14:58:33 +0100 Subject: [PATCH 25/34] Update QDN cache when receiving a metadata file as part of a resource download. --- .../arbitrary/ArbitraryDataFileManager.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 48c41496..69a1150a 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -146,7 +146,7 @@ public class ArbitraryDataFileManager extends Thread { if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) { LOGGER.debug("Requesting data file {} from peer {}", hash58, peer); Long startTime = NTP.getTime(); - ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, null, signature, hash, null); + ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, null, arbitraryTransactionData, signature, hash, null); Long endTime = NTP.getTime(); if (receivedArbitraryDataFile != null) { LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFile.getHash58(), peer, (endTime-startTime)); @@ -191,7 +191,7 @@ public class ArbitraryDataFileManager extends Thread { return receivedAtLeastOneFile; } - private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, Peer requestingPeer, byte[] signature, byte[] hash, Message originalMessage) throws DataException { + private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, Peer requestingPeer, ArbitraryTransactionData arbitraryTransactionData, byte[] signature, byte[] hash, Message originalMessage) throws DataException { ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature); boolean fileAlreadyExists = existingFile.exists(); String hash58 = Base58.encode(hash); @@ -250,6 +250,13 @@ public class ArbitraryDataFileManager extends Thread { } } + // If this is a metadata file then we need to update the cache + if (arbitraryTransactionData != null && arbitraryTransactionData.getMetadataHash() != null) { + if (Arrays.equals(arbitraryTransactionData.getMetadataHash(), hash)) { + ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData); + } + } + return arbitraryDataFile; } @@ -585,7 +592,9 @@ public class ArbitraryDataFileManager extends Thread { // Forward the message to this peer LOGGER.debug("Asking peer {} for hash {}", peerToAsk, hash58); - this.fetchArbitraryDataFile(peerToAsk, peer, signature, hash, message); + // No need to pass arbitraryTransactionData below because this is only used for metadata caching, + // and metadata isn't retained when relaying. + this.fetchArbitraryDataFile(peerToAsk, peer, null, signature, hash, message); } else { LOGGER.debug("Peer {} not found in relay info", peer); From 4b04b99401abaff396790e358fb0d8dffd929167 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 16 Jun 2023 14:59:07 +0100 Subject: [PATCH 26/34] Discard changes before setting status. --- src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index 96fc4e48..aa311786 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -68,6 +68,7 @@ public class ArbitraryDataResource { // Update cache if possible ArbitraryResourceStatus.Status status = arbitraryResourceStatus != null ? arbitraryResourceStatus.getStatus() : null; ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(this.service, this.resourceId, this.identifier); + repository.discardChanges(); repository.getArbitraryRepository().setStatus(arbitraryResourceData, status); repository.saveChanges(); } From badd6ad2b0dbf97bf963e632469fd120aa39ef4d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 23 Jun 2023 11:55:49 +0100 Subject: [PATCH 27/34] Added optional minLevel filter to `GET /arbitrary/resources/search` and the `SEARCH_QDN_RESOURCES` action. --- Q-Apps.md | 1 + .../api/resource/ArbitraryResource.java | 3 ++- .../repository/ArbitraryRepository.java | 2 +- .../hsqldb/HSQLDBArbitraryRepository.java | 21 ++++++++++++++----- src/main/resources/q-apps/q-apps.js | 1 + 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/Q-Apps.md b/Q-Apps.md index 1b4f33bb..bf44a981 100644 --- a/Q-Apps.md +++ b/Q-Apps.md @@ -376,6 +376,7 @@ let res = await qortalRequest({ exactMatchNames: true, // Optional - if true, partial name matches are excluded default: false, // Optional - if true, only resources without identifiers are returned mode: "LATEST", // Optional - whether to return all resources or just the latest for a name/service combination. Possible values: ALL,LATEST. Default: LATEST + minLevel: 1, // Optional - whether to filter results by minimum account level includeStatus: false, // Optional - will take time to respond, so only request if necessary includeMetadata: false, // Optional - will take time to respond, so only request if necessary nameListFilter: "QApp1234Subscriptions", // Optional - will only return results if they are from a name included in supplied list diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index b7eb6cd4..22a74f42 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -172,6 +172,7 @@ public class ArbitraryResource { @Parameter(description = "Exact match names only (if true, partial name matches are excluded)") @QueryParam("exactmatchnames") Boolean exactMatchNamesOnly, @Parameter(description = "Default resources (without identifiers) only") @QueryParam("default") Boolean defaultResource, @Parameter(description = "Search mode") @QueryParam("mode") SearchMode mode, + @Parameter(description = "Min level") @QueryParam("minlevel") Integer minLevel, @Parameter(description = "Filter names by list (exact matches only)") @QueryParam("namefilter") String nameListFilter, @Parameter(description = "Include followed names only") @QueryParam("followedonly") Boolean followedOnly, @Parameter(description = "Exclude blocked content") @QueryParam("excludeblocked") Boolean excludeBlocked, @@ -208,7 +209,7 @@ public class ArbitraryResource { List resources = repository.getArbitraryRepository() .searchArbitraryResources(service, query, identifier, names, title, description, usePrefixOnly, - exactMatchNames, defaultRes, mode, followedOnly, excludeBlocked, includeMetadata, includeStatus, + exactMatchNames, defaultRes, mode, minLevel, followedOnly, excludeBlocked, includeMetadata, includeStatus, before, after, limit, offset, reverse); if (resources == null) { diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 8cff1231..c586871f 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -42,7 +42,7 @@ public interface ArbitraryRepository { public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException; - public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, List namesFilter, boolean defaultResource, SearchMode mode, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException; + public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, List namesFilter, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException; // Arbitrary resources cache save/load diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index a20036de..85c60b4c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -580,8 +580,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { "WHERE name IS NOT NULL"); if (service != null) { - sql.append(" AND service = "); - sql.append(service.value); + sql.append(" AND service = ?"); + bindParams.add(service.value); } if (defaultResource) { @@ -719,7 +719,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { @Override public List searchArbitraryResources(Service service, String query, String identifier, List names, String title, String description, boolean prefixOnly, - List exactMatchNames, boolean defaultResource, SearchMode mode, Boolean followedOnly, Boolean excludeBlocked, + List exactMatchNames, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); @@ -746,11 +746,22 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { break; } + if (minLevel != null) { + // Join tables necessary for level filter + sql.append(" JOIN Names USING (name) JOIN Accounts ON Accounts.account=Names.owner"); + } + sql.append(" LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) WHERE name IS NOT NULL"); + if (minLevel != null) { + // Add level filter + sql.append(" AND Accounts.level >= ?"); + bindParams.add(minLevel); + } + if (service != null) { - sql.append(" AND service = "); - sql.append(service.value); + sql.append(" AND service = ?"); + bindParams.add(service.value); } // Handle general query matches diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js index 6cc1bf08..885a9ba6 100644 --- a/src/main/resources/q-apps/q-apps.js +++ b/src/main/resources/q-apps/q-apps.js @@ -233,6 +233,7 @@ window.addEventListener("message", (event) => { if (data.exactMatchNames != null) url = url.concat("&exactmatchnames=" + new Boolean(data.exactMatchNames).toString()); if (data.default != null) url = url.concat("&default=" + new Boolean(data.default).toString()); if (data.mode != null) url = url.concat("&mode=" + data.mode); + if (data.minLevel != null) url = url.concat("&minlevel=" + data.minLevel); if (data.includeStatus != null) url = url.concat("&includestatus=" + new Boolean(data.includeStatus).toString()); if (data.includeMetadata != null) url = url.concat("&includemetadata=" + new Boolean(data.includeMetadata).toString()); if (data.nameListFilter != null) url = url.concat("&namefilter=" + data.nameListFilter); From c0eeef546ab18b6d3ca2356c641087f6742881c3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 23 Jun 2023 13:30:10 +0100 Subject: [PATCH 28/34] Added support for group encryption in service validation. --- .../org/qortal/arbitrary/misc/Service.java | 5 +++-- .../test/arbitrary/ArbitraryServiceTests.java | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java index 94ca9252..2b8f8d02 100644 --- a/src/main/java/org/qortal/arbitrary/misc/Service.java +++ b/src/main/java/org/qortal/arbitrary/misc/Service.java @@ -186,6 +186,7 @@ public enum Service { private static final ObjectMapper objectMapper = new ObjectMapper(); private static final String encryptedDataPrefix = "qortalEncryptedData"; + private static final String encryptedGroupDataPrefix = "qortalGroupEncryptedData"; Service(int value, boolean requiresValidation, Long maxSize, boolean single, boolean isPrivate, List requiredKeys) { this.value = value; @@ -221,10 +222,10 @@ public enum Service { // Validate private data for single file resources if (this.single) { String dataString = new String(data, StandardCharsets.UTF_8); - if (this.isPrivate && !dataString.startsWith(encryptedDataPrefix)) { + if (this.isPrivate && !dataString.startsWith(encryptedDataPrefix) && !dataString.startsWith(encryptedGroupDataPrefix)) { return ValidationResult.DATA_NOT_ENCRYPTED; } - if (!this.isPrivate && dataString.startsWith(encryptedDataPrefix)) { + if (!this.isPrivate && (dataString.startsWith(encryptedDataPrefix) || dataString.startsWith(encryptedGroupDataPrefix))) { return ValidationResult.DATA_ENCRYPTED; } } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java index 33632b4a..b4c10fac 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java @@ -456,6 +456,25 @@ public class ArbitraryServiceTests extends Common { assertEquals(ValidationResult.OK, service.validate(filePath)); } + @Test + public void testValidPrivateGroupData() throws IOException { + String dataString = "qortalGroupEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1rfla5UBoEcoxbtjgZ9G7FLPb8V/Qfr0bfKWfvMmN06U/pgUdLuv2mGL2V0D3qYd1011MUzGdNG1qERjaCDz8GAi63+KnHHjfMtPgYt6bcqjs4CNV+ZZ4dIt3xxHYyVEBNc="; + + // Write the data a single file in a temp path + Path path = Files.createTempDirectory("testValidPrivateData"); + Path filePath = Paths.get(path.toString(), "test"); + filePath.toFile().deleteOnExit(); + + BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toFile())); + writer.write(dataString); + writer.close(); + + Service service = Service.FILE_PRIVATE; + assertTrue(service.isValidationRequired()); + + assertEquals(ValidationResult.OK, service.validate(filePath)); + } + @Test public void testEncryptedData() throws IOException { String dataString = "qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1rfla5UBoEcoxbtjgZ9G7FLPb8V/Qfr0bfKWfvMmN06U/pgUdLuv2mGL2V0D3qYd1011MUzGdNG1qERjaCDz8GAi63+KnHHjfMtPgYt6bcqjs4CNV+ZZ4dIt3xxHYyVEBNc="; From 537779b1529cb761904be9e8cb3c6dd63d7ef14d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 2 Jul 2023 17:49:49 +0100 Subject: [PATCH 29/34] Use a separate repository instance when updating caches. --- .../arbitrary/ArbitraryDataCacheManager.java | 8 ++++---- .../qortal/transaction/ArbitraryTransaction.java | 14 +++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java index 75b00452..f49fbad2 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java @@ -82,8 +82,8 @@ public class ArbitraryDataCacheManager extends Thread { // Update arbitrary resource caches try { ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); - arbitraryTransaction.updateArbitraryResourceCache(); - arbitraryTransaction.updateArbitraryMetadataCache(); + arbitraryTransaction.updateArbitraryResourceCache(repository); + arbitraryTransaction.updateArbitraryMetadataCache(repository); repository.saveChanges(); LOGGER.debug(() -> String.format("Finished processing transaction %.8s in arbitrary resource queue...", Base58.encode(transactionData.getSignature()))); @@ -166,8 +166,8 @@ public class ArbitraryDataCacheManager extends Thread { // Update arbitrary resource caches ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); - arbitraryTransaction.updateArbitraryResourceCache(); - arbitraryTransaction.updateArbitraryMetadataCache(); + arbitraryTransaction.updateArbitraryResourceCache(repository); + arbitraryTransaction.updateArbitraryMetadataCache(repository); } offset += batchSize; } diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index c0d441e0..851bd7ef 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -28,6 +28,7 @@ import org.qortal.payment.Payment; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.repository.RepositoryManager; import org.qortal.transform.TransformationException; import org.qortal.transform.Transformer; import org.qortal.transform.transaction.ArbitraryTransactionTransformer; @@ -277,7 +278,8 @@ public class ArbitraryTransaction extends Transaction { } private void updateCaches() { - try { + // Very important to use a separate repository instance from the one being used for validation/processing + try (final Repository repository = RepositoryManager.getRepository()) { // If the data is local, we need to perform a few actions if (isDataLocal()) { @@ -288,8 +290,10 @@ public class ArbitraryTransaction extends Transaction { } // Add/update arbitrary resource caches - this.updateArbitraryResourceCache(); - this.updateArbitraryMetadataCache(); + this.updateArbitraryResourceCache(repository); + this.updateArbitraryMetadataCache(repository); + + repository.saveChanges(); } catch (Exception e) { // Log and ignore all exceptions. The cache is updated from other places too, and can be rebuilt if needed. @@ -343,7 +347,7 @@ public class ArbitraryTransaction extends Transaction { * * @throws DataException */ - public void updateArbitraryResourceCache() throws DataException { + public void updateArbitraryResourceCache(Repository repository) throws DataException { // Don't cache resources without a name (such as auto updates) if (arbitraryTransactionData.getName() == null) { return; @@ -412,7 +416,7 @@ public class ArbitraryTransaction extends Transaction { repository.getArbitraryRepository().setStatus(arbitraryResourceData, status); } - public void updateArbitraryMetadataCache() throws DataException { + public void updateArbitraryMetadataCache(Repository repository) throws DataException { // Get the latest transaction ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); if (latestTransactionData == null) { From d8237abde5066c6827108e5030982f4b264e8eb3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 8 Jul 2023 14:16:50 +0100 Subject: [PATCH 30/34] Don't update statuses when processing arbitrary transactions, to improve success rate and speed it up. --- .../arbitrary/ArbitraryDataCacheManager.java | 10 ++++++- .../transaction/ArbitraryTransaction.java | 30 ++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java index f49fbad2..07ae7d67 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java @@ -86,6 +86,10 @@ public class ArbitraryDataCacheManager extends Thread { arbitraryTransaction.updateArbitraryMetadataCache(repository); repository.saveChanges(); + // Update status as separate commit, as this is more prone to failure + arbitraryTransaction.updateArbitraryResourceStatus(repository); + repository.saveChanges(); + LOGGER.debug(() -> String.format("Finished processing transaction %.8s in arbitrary resource queue...", Base58.encode(transactionData.getSignature()))); } catch (DataException e) { @@ -168,11 +172,15 @@ public class ArbitraryDataCacheManager extends Thread { ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); arbitraryTransaction.updateArbitraryResourceCache(repository); arbitraryTransaction.updateArbitraryMetadataCache(repository); + repository.saveChanges(); + + // Update status as separate commit, as this is more prone to failure + arbitraryTransaction.updateArbitraryResourceStatus(repository); + repository.saveChanges(); } offset += batchSize; } - repository.saveChanges(); LOGGER.info("Completed build of arbitrary resources cache."); return true; } diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 851bd7ef..de1dfaad 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -289,7 +289,9 @@ public class ArbitraryTransaction extends Transaction { } } - // Add/update arbitrary resource caches + // Add/update arbitrary resource caches, but don't update the status as this involves time-consuming + // disk reads, and is more prone to failure. The status will be updated on metadata retrieval, or when + // accessing the resource. this.updateArbitraryResourceCache(repository); this.updateArbitraryMetadataCache(repository); @@ -408,6 +410,32 @@ public class ArbitraryTransaction extends Transaction { // Save repository.getArbitraryRepository().save(arbitraryResourceData); + } + + public void updateArbitraryResourceStatus(Repository repository) throws DataException { + // Don't cache resources without a name (such as auto updates) + if (arbitraryTransactionData.getName() == null) { + return; + } + + Service service = arbitraryTransactionData.getService(); + String name = arbitraryTransactionData.getName(); + String identifier = arbitraryTransactionData.getIdentifier(); + + if (service == null) { + // Unsupported service - ignore this resource + return; + } + + // In the cache we store null identifiers as "default", as it is part of the primary key + if (identifier == null) { + identifier = "default"; + } + + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.service = service; + arbitraryResourceData.name = name; + arbitraryResourceData.identifier = identifier; // Update status ArbitraryDataResource resource = new ArbitraryDataResource(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); From 9694094bbf2acae5f1ef5ccb5c5b28f80db085ff Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 8 Jul 2023 14:27:24 +0100 Subject: [PATCH 31/34] Sanitize inputs used for the working path when building arbitrary data, and throw/handle an exception if it still doesn't work. Should fix issue on Windows systems due to reserved characters in certain resource names. --- .../api/gateway/resource/GatewayResource.java | 2 +- .../org/qortal/api/resource/ArbitraryResource.java | 4 ++-- .../org/qortal/arbitrary/ArbitraryDataReader.java | 11 ++++++++--- .../org/qortal/arbitrary/ArbitraryDataRenderer.java | 6 ++++-- .../org/qortal/arbitrary/ArbitraryDataResource.java | 13 +++++++++---- .../arbitrary/ArbitraryDataTransactionBuilder.java | 3 ++- .../org/qortal/utils/ArbitraryTransactionUtils.java | 2 +- src/main/java/org/qortal/utils/StringUtils.java | 13 +++++++++++++ 8 files changed, 40 insertions(+), 14 deletions(-) create mode 100644 src/main/java/org/qortal/utils/StringUtils.java diff --git a/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java b/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java index d82fb98d..bf4e1ea6 100644 --- a/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java +++ b/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java @@ -40,8 +40,8 @@ public class GatewayResource { // If "build=true" has been specified in the query string, build the resource before returning its status if (build != null && build == true) { - ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null); try { + ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null); if (!reader.isBuilding()) { reader.loadSynchronously(false); } diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 22a74f42..9be74a2f 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1287,8 +1287,8 @@ public class ArbitraryResource { private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { - ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); try { + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); int attempts = 0; if (maxAttempts == null) { @@ -1394,8 +1394,8 @@ public class ArbitraryResource { } private FileProperties getFileProperties(Service service, String name, String identifier) { - ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); try { + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); arbitraryDataReader.loadSynchronously(false); java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); if (outputPath == null) { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index f281cec5..8896f44f 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -66,7 +66,7 @@ public class ArbitraryDataReader { // TODO: all builds could be handled by the build queue (even synchronous ones), to avoid the need for this private static Map inProgress = Collections.synchronizedMap(new HashMap<>()); - public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) { + public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) throws DataException { // Ensure names are always lowercase if (resourceIdType == ResourceIdType.NAME) { resourceId = resourceId.toLowerCase(); @@ -90,11 +90,16 @@ public class ArbitraryDataReader { this.canRequestMissingFiles = true; } - private Path buildWorkingPath() { + private Path buildWorkingPath() throws DataException { // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware String baseDir = Settings.getInstance().getTempDataPath(); String identifier = this.identifier != null ? this.identifier : "default"; - return Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier); + + try { + return Paths.get(baseDir, "reader", this.resourceIdType.toString(), StringUtils.sanitizeString(this.resourceId), this.service.toString(), StringUtils.sanitizeString(identifier)); + } catch (InvalidPathException e) { + throw new DataException(String.format("Invalid path: %s", e.getMessage())); + } } public boolean isCachedDataAvailable() { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java index 704533c8..0936b3ec 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java @@ -76,9 +76,11 @@ public class ArbitraryDataRenderer { return ArbitraryDataRenderer.getResponse(response, 500, "QDN is disabled in settings"); } - ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, identifier); - arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only + ArbitraryDataReader arbitraryDataReader; try { + arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, identifier); + arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only + if (!arbitraryDataReader.isCachedDataAvailable()) { // If async is requested, show a loading screen whilst build is in progress if (async) { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index aa311786..16faf838 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -111,10 +111,15 @@ public class ArbitraryDataResource { } // Firstly check the cache to see if it's already built - ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader( - resourceId, resourceIdType, service, identifier); - if (arbitraryDataReader.isCachedDataAvailable()) { - return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount); + try { + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader( + resourceId, resourceIdType, service, identifier); + if (arbitraryDataReader.isCachedDataAvailable()) { + return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount); + } + } catch (DataException e) { + // Assume no usable data + return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount); } // Check if we have all data locally for this resource diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java index a9dd4fcf..c32734dc 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java @@ -117,8 +117,9 @@ public class ArbitraryDataTransactionBuilder { } private Method determineMethodAutomatically() throws DataException { - ArbitraryDataReader reader = new ArbitraryDataReader(this.name, ResourceIdType.NAME, this.service, this.identifier); + ArbitraryDataReader reader; try { + reader = new ArbitraryDataReader(this.name, ResourceIdType.NAME, this.service, this.identifier); reader.loadSynchronously(true); } catch (Exception e) { // Catch all exceptions if the existing resource cannot be loaded first time diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index bf7e2aa6..65f3436c 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -397,8 +397,8 @@ public class ArbitraryTransactionUtils { // If "build" has been specified, build the resource before returning its status if (build != null && build == true) { - ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); try { + ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); if (!reader.isBuilding()) { reader.loadSynchronously(false); } diff --git a/src/main/java/org/qortal/utils/StringUtils.java b/src/main/java/org/qortal/utils/StringUtils.java new file mode 100644 index 00000000..b3ffcfa6 --- /dev/null +++ b/src/main/java/org/qortal/utils/StringUtils.java @@ -0,0 +1,13 @@ +package org.qortal.utils; + +public class StringUtils { + + public static String sanitizeString(String input) { + String sanitized = input + .replaceAll("[<>:\"/\\\\|?*]", "") // Remove invalid characters + .replaceAll("^\\s+|\\s+$", "") // Trim leading and trailing whitespace + .replaceAll("\\s+", "_"); // Replace consecutive whitespace with underscores + + return sanitized; + } +} From eecd37d6bcb73a56db3342976156bb9a15666372 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 5 Aug 2023 13:10:03 +0100 Subject: [PATCH 32/34] Speed up finding arbitrary transactions. --- .../org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index cac02a9e..c73c80ac 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -1040,6 +1040,11 @@ public class HSQLDBDatabaseUpdates { + "REFERENCES ArbitraryResourcesCache (service, name, identifier) ON DELETE CASCADE)"); // For finding metadata by title. stmt.execute("CREATE INDEX ArbitraryMetadataTitleIndex ON ArbitraryMetadataCache (title)"); + + // For finding arbitrary transactions by service + stmt.execute("CREATE INDEX ArbitraryServiceIndex ON ArbitraryTransactions (service)"); + // For finding arbitrary transactions by identifier + stmt.execute("CREATE INDEX ArbitraryIdentifierIndex ON ArbitraryTransactions (identifier)"); break; default: From e44b38819e9b6989c225c91b4c10dfb4c80a6a9c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 5 Aug 2023 13:17:30 +0100 Subject: [PATCH 33/34] Specify table name in query, to avoid potential ambiguity with indexes. --- .../org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 85c60b4c..66829d92 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -412,7 +412,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { "title, description, category, tag1, tag2, tag3, tag4, tag5 " + "FROM ArbitraryResourcesCache " + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + - "WHERE service = ? AND name = ?"); + "WHERE ArbitraryResourcesCache.service = ? AND ArbitraryResourcesCache.name = ?"); bindParams.add(service.value); bindParams.add(name); From 133848ef50b7f2addff217d52bbbfb9fe669d9ba Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 5 Aug 2023 19:42:30 +0100 Subject: [PATCH 34/34] Speed up status rebuilding by excluding transactions that aren't hosted locally by the node. --- .../arbitrary/ArbitraryDataCacheManager.java | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java index 07ae7d67..0c56769f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java @@ -173,14 +173,13 @@ public class ArbitraryDataCacheManager extends Thread { arbitraryTransaction.updateArbitraryResourceCache(repository); arbitraryTransaction.updateArbitraryMetadataCache(repository); repository.saveChanges(); - - // Update status as separate commit, as this is more prone to failure - arbitraryTransaction.updateArbitraryResourceStatus(repository); - repository.saveChanges(); } offset += batchSize; } + // Now refresh all statuses + refreshArbitraryStatuses(repository); + LOGGER.info("Completed build of arbitrary resources cache."); return true; } @@ -193,4 +192,45 @@ public class ArbitraryDataCacheManager extends Thread { } } + private boolean refreshArbitraryStatuses(Repository repository) throws DataException { + try { + LOGGER.info("Refreshing arbitrary resource statuses for locally hosted transactions..."); + SplashFrame.getInstance().updateStatus("Refreshing statuses - please wait..."); + + final int batchSize = 100; + int offset = 0; + + // Loop through all ARBITRARY transactions, and determine latest state + while (!Controller.isStopping()) { + LOGGER.info("Fetching hosted transactions {} - {}", offset, offset+batchSize-1); + + List hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, batchSize, offset); + if (hostedTransactions.isEmpty()) { + // Complete + break; + } + + // Loop through hosted transactions + for (ArbitraryTransactionData transactionData : hostedTransactions) { + + // Determine status and update cache + ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); + arbitraryTransaction.updateArbitraryResourceStatus(repository); + repository.saveChanges(); + } + offset += batchSize; + } + + LOGGER.info("Completed refresh of arbitrary resource statuses."); + return true; + } + catch (DataException e) { + LOGGER.info("Unable to refresh arbitrary resource statuses: {}. The database may have been left in an inconsistent state.", e.getMessage()); + + // Throw an exception so that the node startup is halted, allowing for a retry next time. + repository.discardChanges(); + throw new DataException("Refresh of arbitrary resource statuses failed."); + } + } + }