From 054860b38db36171df01518a41d67a75f4689936 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 13 Nov 2021 11:35:29 +0000 Subject: [PATCH] Rework of storage policy handling, as the previous implementation didn't handle viewed data properly. This also adds a feature to allow data to be deleted and no longer served once a name has been blacklisted. --- .../ArbitraryDataCleanupManager.java | 6 ++-- .../arbitrary/ArbitraryDataManager.java | 10 +++--- .../ArbitraryDataStorageManager.java | 34 +++++++++++++++++-- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index eaea3adb..823c87d5 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -19,8 +19,6 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; public class ArbitraryDataCleanupManager extends Thread { @@ -138,8 +136,8 @@ public class ArbitraryDataCleanupManager extends Thread { // Check to see if we should be hosting data for this transaction at all - if (!storageManager.shouldStoreDataForName(arbitraryTransactionData.getName())) { - LOGGER.info("Deleting transaction {} because we are no longer storing data for name {}", + if (!storageManager.canStoreDataForName(arbitraryTransactionData.getName())) { + LOGGER.info("Deleting transaction {} because we can't host data for name {}", Base58.encode(arbitraryTransactionData.getSignature()), arbitraryTransactionData.getName()); ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData); continue; diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 9a47d3c3..2d429701 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -181,8 +181,8 @@ public class ArbitraryDataManager extends Thread { } ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData(); - // Skip transactions that we don't need to store data for - if (!storageManager.shouldStoreDataForName(arbitraryTransactionData.getName())) { + // Skip transactions that we don't need to proactively store data for + if (!storageManager.shouldPreFetchDataForName(arbitraryTransactionData.getName())) { iterator.remove(); continue; } @@ -515,8 +515,8 @@ public class ArbitraryDataManager extends Thread { invalidateCache(arbitraryTransactionData); // We may also need to broadcast to the network that we are now hosting files for this transaction, - // but only if these files are in accordance with our storage policy (we may have requested them for viewing only) - if (ArbitraryDataStorageManager.getInstance().shouldStoreDataForName(arbitraryTransactionData.getName())) { + // but only if these files are in accordance with our storage policy + if (ArbitraryDataStorageManager.getInstance().canStoreDataForName(arbitraryTransactionData.getName())) { // Use a null peer address to indicate our own Message newArbitrarySignatureMessage = new ArbitrarySignaturesMessage(null, Arrays.asList(signature)); Network.getInstance().broadcast(broadcastPeer -> newArbitrarySignatureMessage); @@ -588,7 +588,7 @@ public class ArbitraryDataManager extends Thread { if (transactionData instanceof ArbitraryTransactionData) { // Check if we're even allowed to serve data for this transaction - if (ArbitraryDataStorageManager.getInstance().shouldStoreDataForName(transactionData.getName())) { + if (ArbitraryDataStorageManager.getInstance().canStoreDataForName(transactionData.getName())) { byte[] hash = transactionData.getData(); byte[] chunkHashes = transactionData.getChunkHashes(); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index 6e27febe..358ebd79 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -25,9 +25,37 @@ public class ArbitraryDataStorageManager { return instance; } - public boolean shouldStoreDataForName(String name) { + public boolean canStoreDataForName(String name) { + // Check if our storage policy and blacklist allows us to host data for this name + switch (Settings.getInstance().getStoragePolicy()) { + case FOLLOWED_AND_VIEWED: + case ALL: + case VIEWED: + // If the policy includes viewed data, we can host it as long as it's not blacklisted + return !this.isNameInBlacklist(name); + + case FOLLOWED: + // If the policy is for followed data only, we have to be following it + return this.isFollowingName(name); + + // For NONE or all else, we shouldn't host this data + case NONE: + default: + return false; + } + } + + private boolean isNameInBlacklist(String name) { + return ResourceListManager.getInstance().listContains("blacklist", "names", name); + } + + public boolean shouldPreFetchDataForName(String name) { if (name == null) { - return this.shouldStoreDataWithoutName(); + return this.shouldPreFetchDataWithoutName(); + } + // Never fetch data from blacklisted names, even if they are followed + if (this.isNameInBlacklist(name)) { + return false; } switch (Settings.getInstance().getStoragePolicy()) { @@ -45,7 +73,7 @@ public class ArbitraryDataStorageManager { } } - private boolean shouldStoreDataWithoutName() { + private boolean shouldPreFetchDataWithoutName() { switch (Settings.getInstance().getStoragePolicy()) { case ALL: return true;