From 82fa6a4fd856c4755e55209478b78c4fdbc93df5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 1 Feb 2022 22:02:14 +0000 Subject: [PATCH] Include "localChunkCount" and "totalChunkCount" in the GET /arbitrary/resource/status/* API responses. These values are left out of other API endpoints where multiple resources are returned, because calculating the chunk counts is too time consuming. --- .../api/gateway/resource/GatewayResource.java | 2 +- .../api/resource/ArbitraryResource.java | 4 +- .../arbitrary/ArbitraryDataResource.java | 73 ++++++++++++++----- .../arbitrary/ArbitraryResourceStatus.java | 10 ++- .../utils/ArbitraryTransactionUtils.java | 40 +++++++++- 5 files changed, 102 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java b/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java index cee1613f..a73de1fb 100644 --- a/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java +++ b/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java @@ -65,7 +65,7 @@ public class GatewayResource { } ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); - return resource.getStatus(); + return resource.getStatus(false); } diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index d542b89c..c49969c5 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1097,7 +1097,7 @@ public class ArbitraryResource { } ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); - return resource.getStatus(); + return resource.getStatus(false); } private List addStatusToResources(List resources) { @@ -1106,7 +1106,7 @@ public class ArbitraryResource { for (ArbitraryResourceInfo resourceInfo : resources) { ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ResourceIdType.NAME, resourceInfo.service, resourceInfo.identifier); - ArbitraryResourceStatus status = resource.getStatus(); + ArbitraryResourceStatus status = resource.getStatus(true); if (status != null) { resourceInfo.status = status; } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index 7e00e0d0..36bd8f4c 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -38,6 +38,8 @@ public class ArbitraryDataResource { private List transactions; private ArbitraryTransactionData latestPutTransaction; private int layerCount; + private Integer localChunkCount = null; + private Integer totalChunkCount = null; public ArbitraryDataResource(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) { this.resourceId = resourceId.toLowerCase(); @@ -51,50 +53,56 @@ public class ArbitraryDataResource { this.identifier = identifier; } - public ArbitraryResourceStatus getStatus() { + public ArbitraryResourceStatus getStatus(boolean quick) { + // Calculate the chunk counts + // Avoid this for "quick" statuses, to speed things up + if (!quick) { + this.calculateChunkCounts(); + } + if (resourceIdType != ResourceIdType.NAME) { // We only support statuses for resources with a name - return new ArbitraryResourceStatus(Status.UNSUPPORTED); + return new ArbitraryResourceStatus(Status.UNSUPPORTED, this.localChunkCount, this.totalChunkCount); } // Check if the name is blocked if (ResourceListManager.getInstance() .listContains("blockedNames", this.resourceId, false)) { - return new ArbitraryResourceStatus(Status.BLOCKED); + return new ArbitraryResourceStatus(Status.BLOCKED, this.localChunkCount, this.totalChunkCount); } - // Firstly check the cache to see if it's already built - ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader( - resourceId, resourceIdType, service, identifier); - if (arbitraryDataReader.isCachedDataAvailable()) { - return new ArbitraryResourceStatus(Status.READY); - } - - // Next check if there's a build in progress + // Check if a build has failed ArbitraryDataBuildQueueItem queueItem = new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier); - if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) { - return new ArbitraryResourceStatus(Status.BUILDING); + if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) { + return new ArbitraryResourceStatus(Status.BUILD_FAILED, this.localChunkCount, this.totalChunkCount); } - // Check if a build has failed - if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) { - return new ArbitraryResourceStatus(Status.BUILD_FAILED); + // Firstly check the cache to see if it's already built + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader( + resourceId, resourceIdType, service, identifier); + if (arbitraryDataReader.isCachedDataAvailable()) { + return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount); } // Check if we have all data locally for this resource if (!this.allFilesDownloaded()) { if (this.isDownloading()) { - return new ArbitraryResourceStatus(Status.DOWNLOADING); + return new ArbitraryResourceStatus(Status.DOWNLOADING, this.localChunkCount, this.totalChunkCount); } else if (this.isDataPotentiallyAvailable()) { - return new ArbitraryResourceStatus(Status.PUBLISHED); + return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount); } - return new ArbitraryResourceStatus(Status.MISSING_DATA); + return new ArbitraryResourceStatus(Status.MISSING_DATA, this.localChunkCount, this.totalChunkCount); + } + + // Check if there's a build in progress + if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) { + return new ArbitraryResourceStatus(Status.BUILDING, this.localChunkCount, this.totalChunkCount); } // We have all data locally - return new ArbitraryResourceStatus(Status.DOWNLOADED); + return new ArbitraryResourceStatus(Status.DOWNLOADED, this.localChunkCount, this.totalChunkCount); } public boolean delete() { @@ -147,6 +155,12 @@ public class ArbitraryDataResource { } private boolean allFilesDownloaded() { + // Use chunk counts to speed things up if we can + if (this.localChunkCount != null && this.totalChunkCount != null && + this.localChunkCount >= this.totalChunkCount) { + return true; + } + try { this.fetchTransactions(); @@ -165,6 +179,25 @@ public class ArbitraryDataResource { } } + private void calculateChunkCounts() { + try { + this.fetchTransactions(); + + List transactionDataList = new ArrayList<>(this.transactions); + int localChunkCount = 0; + int totalChunkCount = 0; + + for (ArbitraryTransactionData transactionData : transactionDataList) { + localChunkCount += ArbitraryTransactionUtils.ourChunkCount(transactionData); + totalChunkCount += ArbitraryTransactionUtils.totalChunkCount(transactionData); + } + + this.localChunkCount = localChunkCount; + this.totalChunkCount = totalChunkCount; + + } catch (DataException e) {} + } + private boolean isRateLimited() { try { this.fetchTransactions(); diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java index 35b83507..5e6ac055 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java @@ -30,13 +30,21 @@ public class ArbitraryResourceStatus { private String title; private String description; + private Integer localChunkCount; + private Integer totalChunkCount; + public ArbitraryResourceStatus() { } - public ArbitraryResourceStatus(Status status) { + public ArbitraryResourceStatus(Status status, Integer localChunkCount, Integer totalChunkCount) { this.id = status.toString(); this.title = status.title; this.description = status.description; + this.localChunkCount = localChunkCount; + this.totalChunkCount = totalChunkCount; } + public ArbitraryResourceStatus(Status status) { + this(status, null, null); + } } diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index 3221c87b..9b81bd68 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -1,5 +1,6 @@ package org.qortal.utils; +import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataFile; @@ -147,16 +148,49 @@ public class ArbitraryTransactionUtils { byte[] metadataHash = transactionData.getMetadataHash(); byte[] signature = transactionData.getSignature(); - if (metadataHash == null) { - // This file doesn't have any metadata, therefore it has no chunks + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + + // Find the folder containing the files + Path parentPath = arbitraryDataFile.getFilePath().getParent(); + String[] files = parentPath.toFile().list(); + if (files == null) { return 0; } + // Remove the original copy indicator file if it exists + files = ArrayUtils.removeElement(files, ".original"); + + int count = files.length; + + // If the complete file exists (and this transaction has chunks), subtract it from the count + if (arbitraryDataFile.chunkCount() > 0 && arbitraryDataFile.exists()) { + // We are only measuring the individual chunks, not the joined file + count -= 1; + } + + return count; + } + + public static int totalChunkCount(ArbitraryTransactionData transactionData) throws DataException { + if (transactionData == null) { + return 0; + } + + byte[] digest = transactionData.getData(); + byte[] metadataHash = transactionData.getMetadataHash(); + byte[] signature = transactionData.getSignature(); + + if (metadataHash == null) { + // This file doesn't have any metadata, therefore it has a single (complete) chunk + return 1; + } + // Load complete file and chunks ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); arbitraryDataFile.setMetadataHash(metadataHash); - return arbitraryDataFile.chunkCount(); + return arbitraryDataFile.chunkCount() + 1; // +1 for the metadata file } public static boolean isFileRecent(Path filePath, long now, long cleanupAfter) {