From 8218bfd24bcf918caa370c3e58007b469beb657a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 31 Oct 2021 22:38:30 +0000 Subject: [PATCH] Initial implementation of storage policies. - Don't attempt to fetch data for transactions which fall outside of the storage policy - Delete files relating to transactions that are no longer within the scope of the storage policy Note: some additional work needs to be done to ensure that viewed files are deleted when using a storage policy that excludes "VIEWED" content. --- .../ArbitraryDataCleanupManager.java | 23 ++++++++ .../arbitrary/ArbitraryDataManager.java | 53 +++++++++++++++++-- .../ArbitraryDataStorageManager.java | 46 ++++++++++++++++ 3 files changed, 117 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index cad574ea..8dfe27bb 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -69,6 +69,9 @@ public class ArbitraryDataCleanupManager extends Thread { public void run() { Thread.currentThread().setName("Arbitrary Data Cleanup Manager"); + // Keep a reference to the storage manager as we will need this a lot + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + // Paginate queries when fetching arbitrary transactions final int limit = 100; int offset = 0; @@ -134,6 +137,23 @@ public class ArbitraryDataCleanupManager extends Thread { // We have at least 1 chunk or file for this transaction, so we might need to delete them... + // Check to see if we should be hosting data for this transaction at all + if (arbitraryTransactionData.getName() != null) { + if (!storageManager.shouldStoreDataForName(arbitraryTransactionData.getName())) { + LOGGER.info("Deleting transaction {} because we are no longer storing data for name {}", + Base58.encode(arbitraryTransactionData.getSignature()), arbitraryTransactionData.getName()); + ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData); + continue; + } + } + else { + // Transaction has no name associated with it + if (!storageManager.shouldStoreDataWithoutName()) { + ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData); + continue; + } + } + // Check to see if we have had a more recent PUT boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData); if (hasMoreRecentPutTransaction) { @@ -145,6 +165,7 @@ public class ArbitraryDataCleanupManager extends Thread { arbitraryTransactionData.getName(), Base58.encode(signature))); ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData); + continue; } if (completeFileExists && !transactionHasChunks) { @@ -163,6 +184,7 @@ public class ArbitraryDataCleanupManager extends Thread { Base58.encode(arbitraryTransactionData.getSignature()))); ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT); + continue; } if (completeFileExists && !allChunksExist) { @@ -171,6 +193,7 @@ public class ArbitraryDataCleanupManager extends Thread { Base58.encode(arbitraryTransactionData.getSignature()))); ArbitraryTransactionUtils.convertFileToChunks(arbitraryTransactionData, now, STALE_FILE_TIMEOUT); + continue; } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index b1d1e8c3..97cf7a18 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -108,6 +108,9 @@ public class ArbitraryDataManager extends Thread { ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1); arbitraryDataBuildExecutor.execute(new ArbitraryDataBuildManager()); + // Keep a reference to the storage manager as we will need this a lot + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + // Paginate queries when fetching arbitrary transactions final int limit = 100; int offset = 0; @@ -136,8 +139,40 @@ public class ArbitraryDataManager extends Thread { } offset += limit; - // Filter out those that already have local data - signatures.removeIf(signature -> hasLocalData(repository, signature)); + // Loop through signatures and remove ones we don't need to process + Iterator iterator = signatures.iterator(); + while (iterator.hasNext()) { + byte[] signature = (byte[]) iterator.next(); + + ArbitraryTransaction arbitraryTransaction = fetchTransaction(repository, signature); + if (arbitraryTransaction == null) { + // Best not to process this one + iterator.remove(); + continue; + } + ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData(); + + // Skip transactions that we don't need to store data for + if (arbitraryTransactionData.getName() != null) { + if (!storageManager.shouldStoreDataForName(arbitraryTransactionData.getName())) { + iterator.remove(); + continue; + } + } + else { + // Transaction has no name associated with it + if (!storageManager.shouldStoreDataWithoutName()) { + iterator.remove(); + continue; + } + } + + // Remove transactions that we already have local data for + if (hasLocalData(arbitraryTransaction)) { + iterator.remove(); + continue; + } + } if (signatures.isEmpty()) { continue; @@ -180,15 +215,23 @@ public class ArbitraryDataManager extends Thread { this.interrupt(); } - private boolean hasLocalData(final Repository repository, final byte[] signature) { + private ArbitraryTransaction fetchTransaction(final Repository repository, byte[] signature) { try { TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); if (!(transactionData instanceof ArbitraryTransactionData)) - return true; + return null; - ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); + return new ArbitraryTransaction(repository, transactionData); + } catch (DataException e) { + return null; + } + } + + private boolean hasLocalData(ArbitraryTransaction arbitraryTransaction) { + try { return arbitraryTransaction.isDataLocal(); + } catch (DataException e) { LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e); return true; diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index 25c31bb6..ce90f001 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -1,5 +1,8 @@ package org.qortal.controller.arbitrary; +import org.qortal.list.ResourceListManager; +import org.qortal.settings.Settings; + public class ArbitraryDataStorageManager { public enum StoragePolicy { @@ -10,6 +13,49 @@ public class ArbitraryDataStorageManager { NONE } + private static ArbitraryDataStorageManager instance; + public ArbitraryDataStorageManager() { } + + public static ArbitraryDataStorageManager getInstance() { + if (instance == null) + instance = new ArbitraryDataStorageManager(); + + return instance; + } + + public boolean shouldStoreDataForName(String name) { + switch (Settings.getInstance().getStoragePolicy()) { + case FOLLOWED: + case FOLLOWED_AND_VIEWED: + return this.isFollowingName(name); + + case ALL: + return true; + + case NONE: + case VIEWED: + default: + return false; + } + } + + public boolean shouldStoreDataWithoutName() { + switch (Settings.getInstance().getStoragePolicy()) { + case ALL: + return true; + + case NONE: + case VIEWED: + case FOLLOWED: + case FOLLOWED_AND_VIEWED: + default: + return false; + } + } + + private boolean isFollowingName(String name) { + return ResourceListManager.getInstance().listContains("followed", "names", name); + } }