diff --git a/src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java b/src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java index d70eb6fb..f4c10bb2 100644 --- a/src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java +++ b/src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java @@ -7,6 +7,7 @@ import javax.xml.bind.annotation.XmlAccessorType; public class ArbitraryResourceSummary { public enum ArbitraryResourceStatus { + DOWNLOADING, DOWNLOADED, BUILDING, READY, diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 5c487800..54d014ac 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -327,7 +327,7 @@ public class ArbitraryDataReader { if (requested) { message = String.format("Requested missing data for file %s", arbitraryDataFile); } else { - message = String.format("Unable to reissue request for missing file %s due to rate limit. Please try again later.", arbitraryDataFile); + message = String.format("Unable to reissue request for missing file %s for signature %s due to rate limit. Please try again later.", arbitraryDataFile, Base58.encode(transactionData.getSignature())); } } else { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index e4fb9e38..20677d7a 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -5,12 +5,14 @@ import org.qortal.api.model.ArbitraryResourceSummary.ArbitraryResourceStatus; import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; +import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.list.ResourceListManager; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.utils.ArbitraryTransactionUtils; +import org.qortal.utils.NTP; import java.util.ArrayList; import java.util.List; @@ -39,6 +41,12 @@ public class ArbitraryDataResource { return new ArbitraryResourceSummary(ArbitraryResourceStatus.UNSUPPORTED); } + // Check if the name is blacklisted + if (ResourceListManager.getInstance() + .listContains("blacklist", "names", this.resourceId, false)) { + return new ArbitraryResourceSummary(ArbitraryResourceStatus.BLACKLISTED); + } + // Firstly check the cache to see if it's already built ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader( resourceId, resourceIdType, service, identifier); @@ -58,14 +66,11 @@ public class ArbitraryDataResource { return new ArbitraryResourceSummary(ArbitraryResourceStatus.BUILD_FAILED); } - // Check if the name is blacklisted - if (ResourceListManager.getInstance() - .listContains("blacklist", "names", this.resourceId, false)) { - return new ArbitraryResourceSummary(ArbitraryResourceStatus.BLACKLISTED); - } - // Check if we have all data locally for this resource if (!this.allFilesDownloaded()) { + if (this.madeRecentRequest()) { + return new ArbitraryResourceSummary(ArbitraryResourceStatus.DOWNLOADING); + } return new ArbitraryResourceSummary(ArbitraryResourceStatus.MISSING_DATA); } @@ -92,7 +97,55 @@ public class ArbitraryDataResource { } } + private boolean isRateLimited() { + try { + this.fetchTransactions(); + + List transactionDataList = new ArrayList<>(this.transactions); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + if (ArbitraryDataManager.getInstance().isSignatureRateLimited(transactionData.getSignature())) { + return true; + } + } + return true; + + } catch (DataException e) { + return false; + } + } + + private boolean madeRecentRequest() { + try { + this.fetchTransactions(); + Long now = NTP.getTime(); + if (now == null) { + return false; + } + + List transactionDataList = new ArrayList<>(this.transactions); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + long lastRequestTime = ArbitraryDataManager.getInstance().lastRequestForSignature(transactionData.getSignature()); + if (now - lastRequestTime < 30 * 1000L) { + return true; + } + } + return false; + + } catch (DataException e) { + return false; + } + } + + + private void fetchTransactions() throws DataException { + if (this.transactions != null && !this.transactions.isEmpty()) { + // Already fetched + return; + } + try (final Repository repository = RepositoryManager.getRepository()) { // Get the most recent PUT diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 2b0e9936..eecfeb98 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -321,14 +321,22 @@ public class ArbitraryDataManager extends Thread { } long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp; - if (timeSinceLastAttempt > 5 * 60 * 1000L) { - // We haven't tried for at least 5 minutes + if (timeSinceLastAttempt > 10 * 1000L) { + // We haven't tried for at least 10 seconds if (directPeerRequestCount < 5) { // We've made less than 5 total attempts return true; } } + if (timeSinceLastAttempt > 5 * 60 * 1000L) { + // We haven't tried for at least 5 minutes + if (directPeerRequestCount < 10) { + // We've made less than 10 total attempts + return true; + } + } + if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) { // We haven't tried for at least 24 hours return true; @@ -337,6 +345,29 @@ public class ArbitraryDataManager extends Thread { return false; } + public boolean isSignatureRateLimited(byte[] signature) { + String signature58 = Base58.encode(signature); + return !this.shouldMakeFileListRequestForSignature(signature58) + && !this.shouldMakeDirectFileRequestsForSignature(signature58); + } + + public long lastRequestForSignature(byte[] signature) { + String signature58 = Base58.encode(signature); + Triple request = arbitraryDataSignatureRequests.get(signature58); + + if (request == null) { + // Not attempted yet + return 0; + } + + // Extract the components + Long lastAttemptTimestamp = request.getC(); + if (lastAttemptTimestamp != null) { + return lastAttemptTimestamp; + } + return 0; + } + private void addToSignatureRequests(String signature58, boolean incrementNetworkRequests, boolean incrementPeerRequests) { Triple request = arbitraryDataSignatureRequests.get(signature58); Long now = NTP.getTime(); @@ -493,8 +524,8 @@ public class ArbitraryDataManager extends Thread { return; } final long requestMinimumTimestamp = now - ARBITRARY_REQUEST_TIMEOUT; - arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); - arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() < requestMinimumTimestamp); + arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < requestMinimumTimestamp); + arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < requestMinimumTimestamp); } public boolean isResourceCached(String resourceId) {