diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index c617b517..1e243c51 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -86,12 +86,12 @@ public class ArbitraryResource { "- If default is set to true, only resources without identifiers will be returned.", responses = { @ApiResponse( - content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class)) + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class)) ) } ) @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public List getResources( + public List getResources( @QueryParam("service") Service service, @QueryParam("name") String name, @QueryParam("identifier") String identifier, @@ -133,8 +133,9 @@ public class ArbitraryResource { } } - List resources = repository.getArbitraryRepository() - .getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked, limit, offset, reverse); + List resources = repository.getArbitraryRepository() + .getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked, + includeMetadata, limit, offset, reverse); if (resources == null) { return new ArrayList<>(); @@ -143,9 +144,6 @@ public class ArbitraryResource { if (includeStatus != null && includeStatus) { resources = ArbitraryTransactionUtils.addStatusToResources(resources); } - if (includeMetadata != null && includeMetadata) { - resources = ArbitraryTransactionUtils.addMetadataToResources(resources); - } return resources; @@ -161,12 +159,12 @@ public class ArbitraryResource { "If default is set to true, only resources without identifiers will be returned.", responses = { @ApiResponse( - content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class)) + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class)) ) } ) @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public List searchResources( + public List searchResources( @QueryParam("service") Service service, @Parameter(description = "Query (searches both name and identifier fields)") @QueryParam("query") String query, @Parameter(description = "Identifier (searches identifier field only)") @QueryParam("identifier") String identifier, @@ -206,8 +204,9 @@ public class ArbitraryResource { names = null; } - List resources = repository.getArbitraryRepository() - .searchArbitraryResources(service, query, identifier, names, usePrefixOnly, exactMatchNames, defaultRes, followedOnly, excludeBlocked, limit, offset, reverse); + List resources = repository.getArbitraryRepository() + .searchArbitraryResources(service, query, identifier, names, usePrefixOnly, exactMatchNames, + defaultRes, followedOnly, excludeBlocked, includeMetadata, limit, offset, reverse); if (resources == null) { return new ArrayList<>(); @@ -216,9 +215,6 @@ public class ArbitraryResource { if (includeStatus != null && includeStatus) { resources = ArbitraryTransactionUtils.addStatusToResources(resources); } - if (includeMetadata != null && includeMetadata) { - resources = ArbitraryTransactionUtils.addMetadataToResources(resources); - } return resources; @@ -479,21 +475,20 @@ public class ArbitraryResource { summary = "List arbitrary resources hosted by this node", responses = { @ApiResponse( - content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class)) + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class)) ) } ) @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public List getHostedResources( + public List getHostedResources( @HeaderParam(Security.API_KEY_HEADER) String apiKey, @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus, - @Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata, @Parameter(ref = "limit") @QueryParam("limit") Integer limit, @Parameter(ref = "offset") @QueryParam("offset") Integer offset, @QueryParam("query") String query) { Security.checkApiCallAllowed(request); - List resources = new ArrayList<>(); + List resources = new ArrayList<>(); try (final Repository repository = RepositoryManager.getRepository()) { @@ -509,21 +504,18 @@ public class ArbitraryResource { if (transactionData.getService() == null) { continue; } - ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); - arbitraryResourceInfo.name = transactionData.getName(); - arbitraryResourceInfo.service = transactionData.getService(); - arbitraryResourceInfo.identifier = transactionData.getIdentifier(); - if (!resources.contains(arbitraryResourceInfo)) { - resources.add(arbitraryResourceInfo); + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.name = transactionData.getName(); + arbitraryResourceData.service = transactionData.getService(); + arbitraryResourceData.identifier = transactionData.getIdentifier(); + if (!resources.contains(arbitraryResourceData)) { + resources.add(arbitraryResourceData); } } if (includeStatus != null && includeStatus) { resources = ArbitraryTransactionUtils.addStatusToResources(resources); } - if (includeMetadata != null && includeMetadata) { - resources = ArbitraryTransactionUtils.addMetadataToResources(resources); - } return resources; diff --git a/src/main/java/org/qortal/arbitrary/misc/Category.java b/src/main/java/org/qortal/arbitrary/misc/Category.java index d56e3d5d..3a1489b8 100644 --- a/src/main/java/org/qortal/arbitrary/misc/Category.java +++ b/src/main/java/org/qortal/arbitrary/misc/Category.java @@ -67,9 +67,12 @@ public enum Category { /** * Same as valueOf() but with fallback to UNCATEGORIZED if there's no match * @param name - * @return a Category (using UNCATEGORIZED if no match found) + * @return a Category (using UNCATEGORIZED if no match found), or null if null name passed */ public static Category uncategorizedValueOf(String name) { + if (name == null) { + return null; + } try { return Category.valueOf(name); } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index f0bd1ef5..e4944a66 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -47,6 +47,7 @@ import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.naming.NameData; import org.qortal.data.network.PeerData; +import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ChatTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.event.Event; @@ -400,6 +401,10 @@ public class Controller extends Thread { RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl()); RepositoryManager.setRepositoryFactory(repositoryFactory); RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); + + try (final Repository repository = RepositoryManager.getRepository()) { + RepositoryManager.buildInitialArbitraryResourcesCache(repository); + } } catch (DataException e) { // If exception has no cause then repository is in use by some other process. @@ -891,6 +896,7 @@ public class Controller extends Thread { if (now >= transaction.getDeadline()) { LOGGER.debug(() -> String.format("Deleting expired, unconfirmed transaction %s", Base58.encode(transactionData.getSignature()))); repository.getTransactionRepository().delete(transactionData); + this.onExpiredTransaction(transactionData); deletedCount++; } } @@ -1203,6 +1209,21 @@ public class Controller extends Thread { }); } + /** + * Callback for when we've deleted an expired, unconfirmed transaction. + *

+ * @implSpec performs actions in a new thread + */ + public void onExpiredTransaction(TransactionData transactionData) { + this.callbackExecutor.execute(() -> { + + // If this is an ARBITRARY transaction, we may need to update the cache + if (transactionData.getType() == TransactionType.ARBITRARY) { + ArbitraryDataManager.getInstance().onExpiredArbitraryTransaction((ArbitraryTransactionData)transactionData); + } + }); + } + public void onPeerHandshakeCompleted(Peer peer) { // Only send if outbound if (peer.isOutbound()) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 9284e672..e32fcb0f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -14,6 +14,7 @@ import org.qortal.arbitrary.ArbitraryDataResource; import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; @@ -539,6 +540,41 @@ public class ArbitraryDataManager extends Thread { return true; } + public void onExpiredArbitraryTransaction(ArbitraryTransactionData arbitraryTransactionData) { + if (arbitraryTransactionData.getName() == null) { + // No name, so we don't care about this transaction + return; + } + + Service service = arbitraryTransactionData.getService(); + String name = arbitraryTransactionData.getName(); + String identifier = arbitraryTransactionData.getIdentifier(); + + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.service = service; + arbitraryResourceData.name = name; + arbitraryResourceData.identifier = identifier; + + try (final Repository repository = RepositoryManager.getRepository()) { + // Find next oldest transaction (which is now the latest transaction) + ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(name, service, null, identifier); + + if (latestTransactionData == null) { + // There are no transactions anymore, so we can delete from the cache entirely (this deletes metadata too) + repository.getArbitraryRepository().delete(arbitraryResourceData); + } + else { + // We found the next oldest transaction, so we can update the cache + ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, latestTransactionData); + arbitraryTransaction.updateArbitraryResourceCache(); + arbitraryTransaction.updateArbitraryMetadataCache(); + } +; + } catch (DataException e) { + // Not much we can do, so ignore for now + } + } + public int getPowDifficulty() { return this.powDifficulty; } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java index 663bc22a..a496485b 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java @@ -15,6 +15,7 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; +import org.qortal.transaction.ArbitraryTransaction; import org.qortal.utils.Base58; import org.qortal.utils.ListUtils; import org.qortal.utils.NTP; @@ -324,37 +325,44 @@ public class ArbitraryMetadataManager { Triple newEntry = new Triple<>(null, null, request.getC()); arbitraryMetadataRequests.put(message.getId(), newEntry); - ArbitraryTransactionData arbitraryTransactionData = null; - - // Forwarding - if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) { - - // Get transaction info - try (final Repository repository = RepositoryManager.getRepository()) { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (!(transactionData instanceof ArbitraryTransactionData)) - return; - arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while finding arbitrary transaction metadata for peer %s", peer), e); + // Get transaction info + try (final Repository repository = RepositoryManager.getRepository()) { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (!(transactionData instanceof ArbitraryTransactionData)) { + return; } + ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - // Check if the name is blocked - boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName())); - if (!isBlocked) { - Peer requestingPeer = request.getB(); - if (requestingPeer != null) { + // Forwarding + if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) { - ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile()); - forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId()); + // Check if the name is blocked + boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName())); + if (!isBlocked) { + Peer requestingPeer = request.getB(); + if (requestingPeer != null) { - // Forward to requesting peer - LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer); - if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) { - requestingPeer.disconnect("failed to forward arbitrary metadata"); + ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile()); + forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId()); + + // Forward to requesting peer + LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer); + if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) { + requestingPeer.disconnect("failed to forward arbitrary metadata"); + } } } } + + // Update arbitrary resource caches + if (arbitraryTransactionData != null) { + ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, arbitraryTransactionData); + arbitraryTransaction.updateArbitraryResourceCache(); + arbitraryTransaction.updateArbitraryMetadataCache(); + } + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while finding arbitrary transaction metadata for peer %s", peer), e); } } diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java similarity index 80% rename from src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java rename to src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java index a09fc5ff..4f636177 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java @@ -7,7 +7,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import java.util.Objects; @XmlAccessorType(XmlAccessType.FIELD) -public class ArbitraryResourceInfo { +public class ArbitraryResourceData { public String name; public Service service; @@ -15,11 +15,11 @@ public class ArbitraryResourceInfo { public ArbitraryResourceStatus status; public ArbitraryResourceMetadata metadata; - public Long size; + public Integer size; public Long created; public Long updated; - public ArbitraryResourceInfo() { + public ArbitraryResourceData() { } @Override @@ -32,10 +32,10 @@ public class ArbitraryResourceInfo { if (o == this) return true; - if (!(o instanceof ArbitraryResourceInfo)) + if (!(o instanceof ArbitraryResourceData)) return false; - ArbitraryResourceInfo other = (ArbitraryResourceInfo) o; + ArbitraryResourceData other = (ArbitraryResourceData) o; return Objects.equals(this.name, other.name) && Objects.equals(this.service, other.service) && diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java index a6aa6e26..614a8e69 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java @@ -18,6 +18,9 @@ public class ArbitraryResourceMetadata { private List files; private String mimeType; + // Only included when updating database + private ArbitraryResourceData arbitraryResourceData; + public ArbitraryResourceMetadata() { } @@ -60,4 +63,48 @@ public class ArbitraryResourceMetadata { public List getFiles() { return this.files; } + + public void setTitle(String title) { + this.title = title; + } + + public String getTitle() { + return this.title; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDescription() { + return this.description; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public List getTags() { + return this.tags; + } + + public void setCategory(Category category) { + this.category = category; + + // Also set categoryName + if (category != null) { + this.categoryName = category.getName(); + } + } + + public Category getCategory() { + return this.category; + } + + public void setArbitraryResourceData(ArbitraryResourceData arbitraryResourceData) { + this.arbitraryResourceData = arbitraryResourceData; + } + public ArbitraryResourceData getArbitraryResourceData() { + return this.arbitraryResourceData; + } } diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java index b9be8034..0f91c2c2 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java @@ -9,7 +9,7 @@ import java.util.List; public class ArbitraryResourceNameInfo { public String name; - public List resources = new ArrayList<>(); + public List resources = new ArrayList<>(); public ArbitraryResourceNameInfo() { } diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 9d9ed8ce..63729e46 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -1,9 +1,8 @@ package org.qortal.repository; import org.qortal.arbitrary.misc.Service; -import org.qortal.data.arbitrary.ArbitraryResourceInfo; -import org.qortal.data.arbitrary.ArbitraryResourceNameInfo; -import org.qortal.data.network.ArbitraryPeerData; +import org.qortal.data.arbitrary.ArbitraryResourceData; +import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; @@ -11,23 +10,42 @@ import java.util.List; public interface ArbitraryRepository { + // Utils + public boolean isDataLocal(byte[] signature) throws DataException; public byte[] fetchData(byte[] signature) throws DataException; + + // Transaction related + public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException; public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException; public List getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException; + public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException; + public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException; - public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Integer limit, Integer offset, Boolean reverse) throws DataException; + // Resource related - public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, List namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Integer limit, Integer offset, Boolean reverse) throws DataException; + public ArbitraryResourceData getArbitraryResource(Service service, String name, String identifier) throws DataException; - public List getArbitraryResourceCreatorNames(Service service, String identifier, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException; + public List getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException; + public List getArbitraryResources(Service service, String identifier, List names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException; + + public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, List namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException; + + + // Arbitrary resources cache save/load + + public void save(ArbitraryResourceData arbitraryResourceData) throws DataException; + public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException; + + public void save(ArbitraryResourceMetadata metadata) throws DataException; + public void delete(ArbitraryResourceMetadata metadata) throws DataException; } diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index 9d76ccae..404b6b34 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -2,8 +2,16 @@ package org.qortal.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.api.resource.TransactionsResource; +import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryResourceData; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.settings.Settings; +import org.qortal.transaction.ArbitraryTransaction; +import org.qortal.transaction.Transaction; import java.sql.SQLException; +import java.util.List; import java.util.concurrent.TimeoutException; public abstract class RepositoryManager { @@ -56,6 +64,69 @@ public abstract class RepositoryManager { } } + public static boolean buildInitialArbitraryResourcesCache(Repository repository) throws DataException { + if (Settings.getInstance().isLite()) { + // Lite nodes have no blockchain + return false; + } + + try { + // Check if QDNResources table is empty + List resources = repository.getArbitraryRepository().getArbitraryResources(10, 0, false); + if (!resources.isEmpty()) { + // Resources exist in the cache, so assume complete. + // We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so + // we shouldn't ever be left in a partially rebuilt state. + LOGGER.debug("Arbitrary resources cache already built"); + return false; + } + + LOGGER.info("Building arbitrary resources cache..."); + + final int batchSize = 100; + int offset = 0; + + // Loop through all ARBITRARY transactions, and determine latest state + while (!Controller.isStopping()) { + LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1); + + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false); + if (signatures.isEmpty()) { + // Complete + break; + } + + // Expand signatures to transactions + for (byte[] signature : signatures) { + ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository + .getTransactionRepository().fromSignature(signature); + + if (transactionData.getService() == null) { + // Unsupported service - ignore this resource + continue; + } + + // Update arbitrary resource caches + ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); + arbitraryTransaction.updateArbitraryResourceCache(); + arbitraryTransaction.updateArbitraryMetadataCache(); + } + offset += batchSize; + } + + repository.saveChanges(); + LOGGER.info("Completed build of initial arbitrary resources cache."); + return true; + } + catch (DataException e) { + LOGGER.info("Unable to build initial arbitrary resources cache: {}. The database may have been left in an inconsistent state.", e.getMessage()); + + // Throw an exception so that the node startup is halted, allowing for a retry next time. + repository.discardChanges(); + throw new DataException("Build of initial arbitrary resources cache failed."); + } + } + public static void setRequestedCheckpoint(Boolean quick) { quickCheckpointRequested = quick; } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 87841ca9..9c88e39c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -2,10 +2,10 @@ package org.qortal.repository.hsqldb; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.bouncycastle.util.Longs; +import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Service; -import org.qortal.data.arbitrary.ArbitraryResourceInfo; -import org.qortal.data.arbitrary.ArbitraryResourceNameInfo; +import org.qortal.data.arbitrary.ArbitraryResourceData; +import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.data.transaction.BaseTransactionData; @@ -22,6 +22,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; public class HSQLDBArbitraryRepository implements ArbitraryRepository { @@ -41,6 +42,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { return (ArbitraryTransactionData) transactionData; } + + // Utils + @Override public boolean isDataLocal(byte[] signature) throws DataException { ArbitraryTransactionData transactionData = getTransactionData(signature); @@ -113,6 +117,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { return null; } + + // Transaction related + @Override public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException { // Already hashed? Nothing to do @@ -211,8 +218,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } } - @Override - public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException { + private ArbitraryTransactionData getSingleTransaction(String name, Service service, Method method, String identifier, boolean firstNotLast) throws DataException { StringBuilder sql = new StringBuilder(1024); sql.append("SELECT type, reference, signature, creator, created_when, fee, " + @@ -228,7 +234,16 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { sql.append(method.value); } - sql.append("ORDER BY created_when DESC LIMIT 1"); + sql.append(" ORDER BY created_when"); + + if (firstNotLast) { + sql.append(" ASC"); + } + else { + sql.append(" DESC"); + } + + sql.append(" LIMIT 1"); try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value, identifier, identifier)) { if (resultSet == null) @@ -284,13 +299,189 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } @Override - public List getArbitraryResources(Service service, String identifier, List names, - boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, - Integer limit, Integer offset, Boolean reverse) throws DataException { + public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException { + return this.getSingleTransaction(name, service, method, identifier, true); + } + + @Override + public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException { + return this.getSingleTransaction(name, service, method, identifier, false); + } + + + // Resource related + + @Override + public ArbitraryResourceData getArbitraryResource(Service service, String name, String identifier) throws DataException { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); - sql.append("SELECT name, service, identifier, MAX(size) AS max_size FROM ArbitraryTransactions WHERE 1=1"); + // Name is required + if (name == null) { + return null; + } + + sql.append("SELECT name, service, identifier, size, created_when, updated_when, " + + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + + "FROM ArbitraryResourcesCache " + + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + + "WHERE service = ? AND name = ?"); + + bindParams.add(service.value); + bindParams.add(name); + + if (identifier != null) { + sql.append(" AND identifier = ?"); + bindParams.add(identifier); + } + else { + sql.append(" AND identifier IS NULL"); + } + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return null; + + String nameResult = resultSet.getString(1); + Service serviceResult = Service.valueOf(resultSet.getInt(2)); + String identifierResult = resultSet.getString(3); + Integer sizeResult = resultSet.getInt(4); + Long created = resultSet.getLong(5); + Long updated = resultSet.getLong(6); + + // Optional metadata fields + String title = resultSet.getString(7); + String description = resultSet.getString(8); + String category = resultSet.getString(9); + String tag1 = resultSet.getString(10); + String tag2 = resultSet.getString(11); + String tag3 = resultSet.getString(12); + String tag4 = resultSet.getString(13); + String tag5 = resultSet.getString(14); + + if (Objects.equals(identifierResult, "default")) { + // Map "default" back to null. This is optional but probably less confusing than returning "default". + identifierResult = null; + } + + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.name = nameResult; + arbitraryResourceData.service = serviceResult; + arbitraryResourceData.identifier = identifierResult; + arbitraryResourceData.size = sizeResult; + arbitraryResourceData.created = created; + arbitraryResourceData.updated = (updated == 0) ? null : updated; + + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setTitle(title); + metadata.setDescription(description); + metadata.setCategory(Category.uncategorizedValueOf(category)); + + List tags = new ArrayList<>(); + if (tag1 != null) tags.add(tag1); + if (tag2 != null) tags.add(tag2); + if (tag3 != null) tags.add(tag3); + if (tag4 != null) tags.add(tag4); + if (tag5 != null) tags.add(tag5); + metadata.setTags(!tags.isEmpty() ? tags : null); + + arbitraryResourceData.metadata = metadata; + + return arbitraryResourceData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary resource from repository", e); + } + } + @Override + public List getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException { + StringBuilder sql = new StringBuilder(512); + List bindParams = new ArrayList<>(); + + sql.append("SELECT name, service, identifier, size, created_when, updated_when, " + + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + + "FROM ArbitraryResourcesCache " + + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + + "WHERE name IS NOT NULL ORDER BY created_when"); + + if (reverse != null && reverse) { + sql.append(" DESC"); + } + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List arbitraryResources = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return arbitraryResources; + + do { + String nameResult = resultSet.getString(1); + Service serviceResult = Service.valueOf(resultSet.getInt(2)); + String identifierResult = resultSet.getString(3); + Integer sizeResult = resultSet.getInt(4); + Long created = resultSet.getLong(5); + Long updated = resultSet.getLong(6); + + // Optional metadata fields + String title = resultSet.getString(7); + String description = resultSet.getString(8); + String category = resultSet.getString(9); + String tag1 = resultSet.getString(10); + String tag2 = resultSet.getString(11); + String tag3 = resultSet.getString(12); + String tag4 = resultSet.getString(13); + String tag5 = resultSet.getString(14); + + if (Objects.equals(identifierResult, "default")) { + // Map "default" back to null. This is optional but probably less confusing than returning "default". + identifierResult = null; + } + + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.name = nameResult; + arbitraryResourceData.service = serviceResult; + arbitraryResourceData.identifier = identifierResult; + arbitraryResourceData.size = sizeResult; + arbitraryResourceData.created = created; + arbitraryResourceData.updated = (updated == 0) ? null : updated; + + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setTitle(title); + metadata.setDescription(description); + metadata.setCategory(Category.uncategorizedValueOf(category)); + + List tags = new ArrayList<>(); + if (tag1 != null) tags.add(tag1); + if (tag2 != null) tags.add(tag2); + if (tag3 != null) tags.add(tag3); + if (tag4 != null) tags.add(tag4); + if (tag5 != null) tags.add(tag5); + metadata.setTags(!tags.isEmpty() ? tags : null); + + arbitraryResourceData.metadata = metadata; + + arbitraryResources.add(arbitraryResourceData); + } while (resultSet.next()); + + return arbitraryResources; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary resources from repository", e); + } + } + + @Override + public List getArbitraryResources(Service service, String identifier, List names, + boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, + Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException { + StringBuilder sql = new StringBuilder(512); + List bindParams = new ArrayList<>(); + + sql.append("SELECT name, service, identifier, size, created_when, updated_when, " + + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + + "FROM ArbitraryResourcesCache " + + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + + "WHERE name IS NOT NULL"); if (service != null) { sql.append(" AND service = "); @@ -351,7 +542,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } } - sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); + sql.append(" ORDER BY created_when"); if (reverse != null && reverse) { sql.append(" DESC"); @@ -359,49 +550,82 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { HSQLDBRepository.limitOffsetSql(sql, limit, offset); - List arbitraryResources = new ArrayList<>(); + List arbitraryResources = new ArrayList<>(); try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { if (resultSet == null) - return null; + return arbitraryResources; do { String nameResult = resultSet.getString(1); Service serviceResult = Service.valueOf(resultSet.getInt(2)); String identifierResult = resultSet.getString(3); Integer sizeResult = resultSet.getInt(4); + Long created = resultSet.getLong(5); + Long updated = resultSet.getLong(6); - // We should filter out resources without names - if (nameResult == null) { - continue; + // Optional metadata fields + String title = resultSet.getString(7); + String description = resultSet.getString(8); + String category = resultSet.getString(9); + String tag1 = resultSet.getString(10); + String tag2 = resultSet.getString(11); + String tag3 = resultSet.getString(12); + String tag4 = resultSet.getString(13); + String tag5 = resultSet.getString(14); + + if (Objects.equals(identifierResult, "default")) { + // Map "default" back to null. This is optional but probably less confusing than returning "default". + identifierResult = null; } - ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); - arbitraryResourceInfo.name = nameResult; - arbitraryResourceInfo.service = serviceResult; - arbitraryResourceInfo.identifier = identifierResult; - arbitraryResourceInfo.size = Longs.valueOf(sizeResult); + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.name = nameResult; + arbitraryResourceData.service = serviceResult; + arbitraryResourceData.identifier = identifierResult; + arbitraryResourceData.size = sizeResult; + arbitraryResourceData.created = created; + arbitraryResourceData.updated = (updated == 0) ? null : updated; - arbitraryResources.add(arbitraryResourceInfo); + if (includeMetadata != null && includeMetadata) { + // TODO: we could avoid the join altogether + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setTitle(title); + metadata.setDescription(description); + metadata.setCategory(Category.uncategorizedValueOf(category)); + + List tags = new ArrayList<>(); + if (tag1 != null) tags.add(tag1); + if (tag2 != null) tags.add(tag2); + if (tag3 != null) tags.add(tag3); + if (tag4 != null) tags.add(tag4); + if (tag5 != null) tags.add(tag5); + metadata.setTags(!tags.isEmpty() ? tags : null); + + arbitraryResourceData.metadata = metadata; + } + + arbitraryResources.add(arbitraryResourceData); } while (resultSet.next()); return arbitraryResources; } catch (SQLException e) { - throw new DataException("Unable to fetch arbitrary transactions from repository", e); + throw new DataException("Unable to fetch arbitrary resources from repository", e); } } @Override - public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, + public List searchArbitraryResources(Service service, String query, String identifier, List names, boolean prefixOnly, List exactMatchNames, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, - Integer limit, Integer offset, Boolean reverse) throws DataException { + Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(512); List bindParams = new ArrayList<>(); - sql.append("SELECT name, service, identifier, MAX(size) AS max_size, MIN(created_when) AS date_created, MAX(created_when) AS date_updated " + - "FROM ArbitraryTransactions " + - "JOIN Transactions USING (signature) " + - "WHERE 1=1"); + sql.append("SELECT name, service, identifier, size, created_when, updated_when, " + + "title, description, category, tag1, tag2, tag3, tag4, tag5 " + + "FROM ArbitraryResourcesCache " + + "LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " + + "WHERE name IS NOT NULL"); if (service != null) { sql.append(" AND service = "); @@ -492,7 +716,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } } - sql.append(" GROUP BY name, service, identifier ORDER BY date_created"); + sql.append(" ORDER BY created_when"); if (reverse != null && reverse) { sql.append(" DESC"); @@ -500,98 +724,156 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { HSQLDBRepository.limitOffsetSql(sql, limit, offset); - List arbitraryResources = new ArrayList<>(); + List arbitraryResources = new ArrayList<>(); try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { if (resultSet == null) - return null; + return arbitraryResources; do { String nameResult = resultSet.getString(1); Service serviceResult = Service.valueOf(resultSet.getInt(2)); String identifierResult = resultSet.getString(3); Integer sizeResult = resultSet.getInt(4); - long dateCreated = resultSet.getLong(5); - long dateUpdated = resultSet.getLong(6); + Long created = resultSet.getLong(5); + Long updated = resultSet.getLong(6); - // We should filter out resources without names - if (nameResult == null) { - continue; + // Optional metadata fields + String title = resultSet.getString(7); + String description = resultSet.getString(8); + String category = resultSet.getString(9); + String tag1 = resultSet.getString(10); + String tag2 = resultSet.getString(11); + String tag3 = resultSet.getString(12); + String tag4 = resultSet.getString(13); + String tag5 = resultSet.getString(14); + + if (Objects.equals(identifierResult, "default")) { + // Map "default" back to null. This is optional but probably less confusing than returning "default". + identifierResult = null; } - ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); - arbitraryResourceInfo.name = nameResult; - arbitraryResourceInfo.service = serviceResult; - arbitraryResourceInfo.identifier = identifierResult; - arbitraryResourceInfo.size = Longs.valueOf(sizeResult); - arbitraryResourceInfo.created = dateCreated; - arbitraryResourceInfo.updated = dateUpdated; + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.name = nameResult; + arbitraryResourceData.service = serviceResult; + arbitraryResourceData.identifier = identifierResult; + arbitraryResourceData.size = sizeResult; + arbitraryResourceData.created = created; + arbitraryResourceData.updated = (updated == 0) ? null : updated; - arbitraryResources.add(arbitraryResourceInfo); + if (includeMetadata != null && includeMetadata) { + // TODO: we could avoid the join altogether + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setTitle(title); + metadata.setDescription(description); + metadata.setCategory(Category.uncategorizedValueOf(category)); + + List tags = new ArrayList<>(); + if (tag1 != null) tags.add(tag1); + if (tag2 != null) tags.add(tag2); + if (tag3 != null) tags.add(tag3); + if (tag4 != null) tags.add(tag4); + if (tag5 != null) tags.add(tag5); + metadata.setTags(!tags.isEmpty() ? tags : null); + + arbitraryResourceData.metadata = metadata; + } + + arbitraryResources.add(arbitraryResourceData); } while (resultSet.next()); return arbitraryResources; } catch (SQLException e) { - throw new DataException("Unable to fetch arbitrary transactions from repository", e); + throw new DataException("Unable to fetch arbitrary resources from repository", e); + } + } + + + // Arbitrary resources cache save/load + + @Override + public void save(ArbitraryResourceData arbitraryResourceData) throws DataException { + HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryResourcesCache"); + + saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name) + .bind("identifier", arbitraryResourceData.identifier).bind("size", arbitraryResourceData.size) + .bind("created_when", arbitraryResourceData.created).bind("updated_when", arbitraryResourceData.updated); + + try { + saveHelper.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save arbitrary resource info into repository", e); } } @Override - public List getArbitraryResourceCreatorNames(Service service, String identifier, - boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException { - StringBuilder sql = new StringBuilder(512); + public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException { + // NOTE: arbitrary metadata are deleted automatically by the database thanks to "ON DELETE CASCADE" + // in ArbitraryMetadataCache' FOREIGN KEY definition. + try { + this.repository.delete("ArbitraryResourcesCache", "service = ? AND name = ? AND identifier = ?", + arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier); - sql.append("SELECT name FROM ArbitraryTransactions WHERE 1=1"); - - if (service != null) { - sql.append(" AND service = "); - sql.append(service.value); - } - - if (defaultResource) { - // Default resource requested - use NULL identifier - // The AND ? IS NULL AND ? IS NULL is a hack to make use of the identifier params in checkedExecute() - identifier = null; - sql.append(" AND (identifier IS NULL AND ? IS NULL AND ? IS NULL)"); - } - else { - // Non-default resource requested - // Use an exact match identifier, or list all if supplied identifier is null - sql.append(" AND (identifier = ? OR (? IS NULL))"); - } - - sql.append(" GROUP BY name ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); - - if (reverse != null && reverse) { - sql.append(" DESC"); - } - - HSQLDBRepository.limitOffsetSql(sql, limit, offset); - - List arbitraryResources = new ArrayList<>(); - - try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), identifier, identifier)) { - if (resultSet == null) - return null; - - do { - String name = resultSet.getString(1); - - // We should filter out resources without names - if (name == null) { - continue; - } - - ArbitraryResourceNameInfo arbitraryResourceNameInfo = new ArbitraryResourceNameInfo(); - arbitraryResourceNameInfo.name = name; - - arbitraryResources.add(arbitraryResourceNameInfo); - } while (resultSet.next()); - - return arbitraryResources; } catch (SQLException e) { - throw new DataException("Unable to fetch arbitrary transactions from repository", e); + throw new DataException("Unable to delete account from repository", e); } } + + /* Arbitrary metadata cache */ + + @Override + public void save(ArbitraryResourceMetadata metadata) throws DataException { + HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryMetadataCache"); + + ArbitraryResourceData arbitraryResourceData = metadata.getArbitraryResourceData(); + if (arbitraryResourceData == null) { + throw new DataException("Can't save metadata without a referenced resource"); + } + + String tag1 = null; + String tag2 = null; + String tag3 = null; + String tag4 = null; + String tag5 = null; + + List tags = metadata.getTags(); + if (tags != null) { + if (tags.size() > 0) tag1 = tags.get(0); + if (tags.size() > 1) tag2 = tags.get(1); + if (tags.size() > 2) tag3 = tags.get(2); + if (tags.size() > 3) tag4 = tags.get(3); + if (tags.size() > 4) tag5 = tags.get(4); + } + + String category = metadata.getCategory() != null ? metadata.getCategory().toString() : null; + + saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name) + .bind("identifier", arbitraryResourceData.identifier).bind("title", metadata.getTitle()) + .bind("description", metadata.getDescription()).bind("category", category) + .bind("tag1", tag1).bind("tag2", tag2).bind("tag3", tag3).bind("tag4", tag4) + .bind("tag5", tag5); + + try { + saveHelper.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save arbitrary metadata into repository", e); + } + } + + @Override + public void delete(ArbitraryResourceMetadata metadata) throws DataException { + ArbitraryResourceData arbitraryResourceData = metadata.getArbitraryResourceData(); + if (arbitraryResourceData == null) { + throw new DataException("Can't delete metadata without a referenced resource"); + } + + try { + this.repository.delete("ArbitraryMetadataCache", "service = ? AND name = ? AND identifier = ?", + arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier); + + } catch (SQLException e) { + throw new DataException("Unable to delete account from repository", e); + } + } } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index aecac034..e3c705e8 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -901,7 +901,7 @@ public class HSQLDBDatabaseUpdates { case 37: // ARBITRARY transaction updates for off-chain data storage - // We may want to use a nonce rather than a transaction fee on the data chain + // We may want to use a nonce rather than a transaction fee for ARBITRARY transactions stmt.execute("ALTER TABLE ArbitraryTransactions ADD nonce INT NOT NULL DEFAULT 0"); // We need to know the total size of the data file(s) associated with each transaction stmt.execute("ALTER TABLE ArbitraryTransactions ADD size INT NOT NULL DEFAULT 0"); @@ -993,6 +993,44 @@ public class HSQLDBDatabaseUpdates { stmt.execute("ALTER TABLE CancelSellNameTransactions ADD sale_price QortalAmount"); break; + case 47: + // We need to keep a local cache of arbitrary resources (items published to QDN), for easier searching. + // IMPORTANT: this is a cache of the last known state of a resource (both confirmed + // and valid unconfirmed). It cannot be assumed that all nodes will contain the same state at a + // given block height, and therefore must NOT be used for any consensus/validation code. It is + // simply a cache, to avoid having to query the raw transactions and the metadata in flat files + // when serving API requests. + // ARBITRARY transactions aren't really suitable for updating resources in the same way we'd update + // names or groups for instance, as there is no distinction between creations and updates, and metadata + // is off-chain. Plus, QDN allows (valid) unconfirmed data to be queried and viewed. It is very + // easy to keep a cache of the latest transaction's data, but anything more than that would need + // considerable thought (and most likely a rewrite). + + stmt.execute("CREATE TABLE ArbitraryResourcesCache (service SMALLINT NOT NULL, " + + "name RegisteredName NOT NULL, identifier VARCHAR(64), size INT NOT NULL, " + + "created_when EpochMillis NOT NULL, updated_when EpochMillis, " + + "PRIMARY KEY (service, name, identifier))"); + // For finding resources by service. + stmt.execute("CREATE INDEX ArbitraryResourcesServiceIndex ON ArbitraryResourcesCache (service)"); + // For finding resources by name. + stmt.execute("CREATE INDEX ArbitraryResourcesNameIndex ON ArbitraryResourcesCache (name)"); + // For finding resources by identifier. + stmt.execute("CREATE INDEX ArbitraryResourcesIdentifierIndex ON ArbitraryResourcesCache (identifier)"); + // For finding resources by creation date (the default column when ordering). + stmt.execute("CREATE INDEX ArbitraryResourcesCreatedIndex ON ArbitraryResourcesCache (created_when)"); + // Use a separate table space as this table will be very large. + stmt.execute("SET TABLE ArbitraryResourcesCache NEW SPACE"); + + stmt.execute("CREATE TABLE ArbitraryMetadataCache (service SMALLINT NOT NULL, " + + "name RegisteredName NOT NULL, identifier VARCHAR(64), " + + "title VARCHAR(80), description VARCHAR(240), category VARCHAR(64), " + + "tag1 VARCHAR(20), tag2 VARCHAR(20), tag3 VARCHAR(20), tag4 VARCHAR(20), tag5 VARCHAR(20), " + + "PRIMARY KEY (service, name, identifier), FOREIGN KEY (service, name, identifier) " + + "REFERENCES ArbitraryResourcesCache (service, name, identifier) ON DELETE CASCADE)"); + // For finding metadata by title. + stmt.execute("CREATE INDEX ArbitraryMetadataTitleIndex ON ArbitraryMetadataCache (title)"); + break; + default: // nothing to do return false; diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 7034d7b8..88be95a2 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -1,18 +1,21 @@ package org.qortal.transaction; -import java.util.Arrays; +import java.io.IOException; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; import org.qortal.account.Account; +import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; +import org.qortal.arbitrary.misc.Service; import org.qortal.block.BlockChain; import org.qortal.controller.arbitrary.ArbitraryDataManager; -import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.crypto.Crypto; import org.qortal.crypto.MemoryPoW; import org.qortal.data.PaymentData; +import org.qortal.data.arbitrary.ArbitraryResourceData; +import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; @@ -248,6 +251,10 @@ public class ArbitraryTransaction extends Transaction { ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData); } } + + // Add to arbitrary resource caches + this.updateArbitraryResourceCache(); + this.updateArbitraryMetadataCache(); } @Override @@ -304,4 +311,123 @@ public class ArbitraryTransaction extends Transaction { return null; } + /** + * Update the arbitrary resources cache. + * This finds the latest transaction and replaces the + * majority of the data in the cache. The current + * transaction is used for the created time, + * if it has a lower timestamp than the existing value. + * It's also used to identify the correct + * service/name/identifier combination. + * + * @throws DataException + */ + public void updateArbitraryResourceCache() throws DataException { + // Don't cache resources without a name (such as auto updates) + if (arbitraryTransactionData.getName() == null) { + return; + } + + // Get the latest transaction + ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); + if (latestTransactionData == null) { + // We don't have a latest transaction, so give up + return; + } + + Service service = arbitraryTransactionData.getService(); + String name = arbitraryTransactionData.getName(); + String identifier = arbitraryTransactionData.getIdentifier(); + + // In the cache we store null identifiers as "default", as it is part of the primary key + if (identifier == null) { + identifier = "default"; + } + + // Get existing cached entry if it exists + ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository() + .getArbitraryResource(service, name, identifier); + + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.service = service; + arbitraryResourceData.name = name; + arbitraryResourceData.identifier = identifier; + + // Check for existing cached data + if (existingArbitraryResourceData == null) { + // Nothing exists yet, so set everything from the newest transaction + arbitraryResourceData.created = latestTransactionData.getTimestamp(); + arbitraryResourceData.updated = null; + } + else { + // An entry already exists - update created time from current transaction if this is older + arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp()); + + // Set updated time to the latest transaction's timestamp, unless it matches the creation time + if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) { + // Latest transaction matches created time, so it hasn't been updated + arbitraryResourceData.updated = null; + } + else { + arbitraryResourceData.updated = latestTransactionData.getTimestamp(); + } + } + + arbitraryResourceData.size = latestTransactionData.getSize(); + + // Save + repository.getArbitraryRepository().save(arbitraryResourceData); + } + + public void updateArbitraryMetadataCache() throws DataException { + // Get the latest transaction + ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); + if (latestTransactionData == null) { + // We don't have a latest transaction, so give up + return; + } + + Service service = latestTransactionData.getService(); + String name = latestTransactionData.getName(); + String identifier = latestTransactionData.getIdentifier(); + + // In the cache we store null identifiers as "default", as it is part of the primary key + if (identifier == null) { + identifier = "default"; + } + + ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); + arbitraryResourceData.service = service; + arbitraryResourceData.name = name; + arbitraryResourceData.identifier = identifier; + + // Update metadata for latest transaction if it is local + if (latestTransactionData.getMetadataHash() != null) { + ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature()); + if (metadataFile.exists()) { + ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath()); + try { + transactionMetadata.read(); + + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setArbitraryResourceData(arbitraryResourceData); + metadata.setTitle(transactionMetadata.getTitle()); + metadata.setDescription(transactionMetadata.getDescription()); + metadata.setCategory(transactionMetadata.getCategory()); + metadata.setTags(transactionMetadata.getTags()); + repository.getArbitraryRepository().save(metadata); + + } catch (IOException e) { + // Ignore, as we can add it again later + } + } else { + // We don't have a local copy of this metadata file, so delete it from the cache + // It will be re-added if the file later arrives via the network + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setArbitraryResourceData(arbitraryResourceData); + repository.getArbitraryRepository().delete(metadata); + } + } + } + } diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index efd84110..15ade240 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -4,10 +4,8 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.*; -import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Service; -import org.qortal.data.arbitrary.ArbitraryResourceInfo; -import org.qortal.data.arbitrary.ArbitraryResourceMetadata; +import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceStatus; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; @@ -258,8 +256,7 @@ public class ArbitraryTransactionUtils { "chunks if needed", Base58.encode(completeHash)); ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, cleanupAfter); - } - else { + } else { // File might be in use. It's best to leave it and it it will be cleaned up later. } } @@ -271,6 +268,7 @@ public class ArbitraryTransactionUtils { * When first uploaded, files go into a _misc folder as they are not yet associated with a * transaction signature. Once the transaction is broadcast, they need to be moved to the * correct location, keyed by the transaction signature. + * * @param arbitraryTransactionData * @return * @throws DataException @@ -356,8 +354,7 @@ public class ArbitraryTransactionUtils { file.createNewFile(); } } - } - catch (DataException | IOException e) { + } catch (DataException | IOException e) { LOGGER.info("Unable to check and relocate all files for signature {}: {}", Base58.encode(arbitraryTransactionData.getSignature()), e.getMessage()); } @@ -366,7 +363,7 @@ public class ArbitraryTransactionUtils { } public static List limitOffsetTransactions(List transactions, - Integer limit, Integer offset) { + Integer limit, Integer offset) { if (limit != null && limit == 0) { limit = null; } @@ -389,6 +386,7 @@ public class ArbitraryTransactionUtils { /** * Lookup status of resource + * * @param service * @param name * @param identifier @@ -413,10 +411,10 @@ public class ArbitraryTransactionUtils { return resource.getStatus(false); } - public static List addStatusToResources(List resources) { + public static List addStatusToResources(List resources) { // Determine and add the status of each resource - List updatedResources = new ArrayList<>(); - for (ArbitraryResourceInfo resourceInfo : resources) { + List updatedResources = new ArrayList<>(); + for (ArbitraryResourceData resourceInfo : resources) { try { ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME, resourceInfo.service, resourceInfo.identifier); @@ -433,21 +431,4 @@ public class ArbitraryTransactionUtils { } return updatedResources; } - - public static List addMetadataToResources(List resources) { - // Add metadata fields to each resource if they exist - List updatedResources = new ArrayList<>(); - for (ArbitraryResourceInfo resourceInfo : resources) { - ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME, - resourceInfo.service, resourceInfo.identifier); - ArbitraryDataTransactionMetadata transactionMetadata = resource.getLatestTransactionMetadata(); - ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(transactionMetadata, false); - if (resourceMetadata != null) { - resourceInfo.metadata = resourceMetadata; - } - updatedResources.add(resourceInfo); - } - return updatedResources; - } - }