Initial implementation of storage policies.

- Don't attempt to fetch data for transactions which fall outside of the storage policy
- Delete files relating to transactions that are no longer within the scope of the storage policy

Note: some additional work needs to be done to ensure that viewed files are deleted when using a storage policy that excludes "VIEWED" content.

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java

@@ -69,6 +69,9 @@ public class ArbitraryDataCleanupManager extends Thread {
 	public void run() {
 		Thread.currentThread().setName("Arbitrary Data Cleanup Manager");
 
+		// Keep a reference to the storage manager as we will need this a lot
+		ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
+
 		// Paginate queries when fetching arbitrary transactions
 		final int limit = 100;
 		int offset = 0;
@@ -134,6 +137,23 @@ public class ArbitraryDataCleanupManager extends Thread {
 						// We have at least 1 chunk or file for this transaction, so we might need to delete them...
 
 
+						// Check to see if we should be hosting data for this transaction at all
+						if (arbitraryTransactionData.getName() != null) {
+							if (!storageManager.shouldStoreDataForName(arbitraryTransactionData.getName())) {
+								LOGGER.info("Deleting transaction {} because we are no longer storing data for name {}",
+										Base58.encode(arbitraryTransactionData.getSignature()), arbitraryTransactionData.getName());
+								ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
+								continue;
+							}
+						}
+						else {
+							// Transaction has no name associated with it
+							if (!storageManager.shouldStoreDataWithoutName()) {
+								ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
+								continue;
+							}
+						}
+
 						// Check to see if we have had a more recent PUT
 						boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
 						if (hasMoreRecentPutTransaction) {
@@ -145,6 +165,7 @@ public class ArbitraryDataCleanupManager extends Thread {
 									arbitraryTransactionData.getName(), Base58.encode(signature)));
 
 							ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
+							continue;
 						}
 
 						if (completeFileExists && !transactionHasChunks) {
@@ -163,6 +184,7 @@ public class ArbitraryDataCleanupManager extends Thread {
 									Base58.encode(arbitraryTransactionData.getSignature())));
 
 							ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
+							continue;
 						}
 
 						if (completeFileExists && !allChunksExist) {
@@ -171,6 +193,7 @@ public class ArbitraryDataCleanupManager extends Thread {
 									Base58.encode(arbitraryTransactionData.getSignature())));
 
 							ArbitraryTransactionUtils.convertFileToChunks(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
+							continue;
 						}
 					}
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java

@@ -108,6 +108,9 @@ public class ArbitraryDataManager extends Thread {
 		ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1);
 		arbitraryDataBuildExecutor.execute(new ArbitraryDataBuildManager());
 
+		// Keep a reference to the storage manager as we will need this a lot
+		ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
+
 		// Paginate queries when fetching arbitrary transactions
 		final int limit = 100;
 		int offset = 0;
@@ -136,8 +139,40 @@ public class ArbitraryDataManager extends Thread {
 					}
 					offset += limit;
 
-					// Filter out those that already have local data
+					// Loop through signatures and remove ones we don't need to process
-					signatures.removeIf(signature -> hasLocalData(repository, signature));
+					Iterator iterator = signatures.iterator();
+					while (iterator.hasNext()) {
+						byte[] signature = (byte[]) iterator.next();
+
+						ArbitraryTransaction arbitraryTransaction = fetchTransaction(repository, signature);
+						if (arbitraryTransaction == null) {
+							// Best not to process this one
+							iterator.remove();
+							continue;
+						}
+						ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
+
+						// Skip transactions that we don't need to store data for
+						if (arbitraryTransactionData.getName() != null) {
+							if (!storageManager.shouldStoreDataForName(arbitraryTransactionData.getName())) {
+								iterator.remove();
+								continue;
+							}
+						}
+						else {
+							// Transaction has no name associated with it
+							if (!storageManager.shouldStoreDataWithoutName()) {
+								iterator.remove();
+								continue;
+							}
+						}
+
+						// Remove transactions that we already have local data for
+						if (hasLocalData(arbitraryTransaction)) {
+							iterator.remove();
+							continue;
+						}
+					}
 
 					if (signatures.isEmpty()) {
 						continue;
@@ -180,15 +215,23 @@ public class ArbitraryDataManager extends Thread {
 		this.interrupt();
 	}
 
-	private boolean hasLocalData(final Repository repository, final byte[] signature) {
+	private ArbitraryTransaction fetchTransaction(final Repository repository, byte[] signature) {
 		try {
 			TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
 			if (!(transactionData instanceof ArbitraryTransactionData))
-				return true;
+				return null;
 
-			ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
+			return new ArbitraryTransaction(repository, transactionData);
 
+		} catch (DataException e) {
+			return null;
+		}
+	}
+
+	private boolean hasLocalData(ArbitraryTransaction arbitraryTransaction) {
+		try {
 			return arbitraryTransaction.isDataLocal();
+
 		} catch (DataException e) {
 			LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e);
 			return true;

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java

@@ -1,5 +1,8 @@
 package org.qortal.controller.arbitrary;
 
+import org.qortal.list.ResourceListManager;
+import org.qortal.settings.Settings;
+
 public class ArbitraryDataStorageManager {
 
     public enum StoragePolicy {
@@ -10,6 +13,49 @@ public class ArbitraryDataStorageManager {
         NONE
     }
 
+    private static ArbitraryDataStorageManager instance;
+
     public ArbitraryDataStorageManager() {
     }
+
+    public static ArbitraryDataStorageManager getInstance() {
+        if (instance == null)
+            instance = new ArbitraryDataStorageManager();
+
+        return instance;
+    }
+
+    public boolean shouldStoreDataForName(String name) {
+        switch (Settings.getInstance().getStoragePolicy()) {
+            case FOLLOWED:
+            case FOLLOWED_AND_VIEWED:
+                return this.isFollowingName(name);
+                
+            case ALL:
+                return true;
+
+            case NONE:
+            case VIEWED:
+            default:
+                return false;
+        }
+    }
+
+    public boolean shouldStoreDataWithoutName() {
+        switch (Settings.getInstance().getStoragePolicy()) {
+            case ALL:
+                return true;
+
+            case NONE:
+            case VIEWED:
+            case FOLLOWED:
+            case FOLLOWED_AND_VIEWED:
+            default:
+                return false;
+        }
+    }
+
+    private boolean isFollowingName(String name) {
+        return ResourceListManager.getInstance().listContains("followed", "names", name);
+    }
 }