From c6e5c4e3b52391c0acf0bea07bfb86856048cf2e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 5 Dec 2021 16:36:54 +0000 Subject: [PATCH] Tidy up of storage thresholds and comments. Also reduced deletion threshold from 100% to 98% to reduce the chances of filling up the disk and corrupting the db. --- .../ArbitraryDataCleanupManager.java | 20 ++++++++++--------- .../ArbitraryDataStorageManager.java | 9 +++++++-- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index 768b3344..5e77e5c5 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -23,6 +23,8 @@ import java.nio.file.Paths; import java.security.SecureRandom; import java.util.*; +import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD; + public class ArbitraryDataCleanupManager extends Thread { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCleanupManager.class); @@ -51,7 +53,7 @@ public class ArbitraryDataCleanupManager extends Thread { /* TODO: - - Delete old files not associated with transactions + - Delete files from the _misc folder once they reach a certain age */ @@ -208,8 +210,10 @@ public class ArbitraryDataCleanupManager extends Thread { this.checkForExpiredTransactions(repository); // Delete additional data at random if we're over our storage limit - // Use a threshold of 1, for the same reasons as above - if (!storageManager.isStorageSpaceAvailable(1.0f)) { + // Use the DELETION_THRESHOLD so that we only start deleting once the hard limit is reached + // This also allows some headroom between the regular threshold (90%) and the hard + // limit, to avoid data getting into a fetch/delete loop. + if (!storageManager.isStorageSpaceAvailable(DELETION_THRESHOLD)) { // Rate limit, to avoid repeated calls to calculateDirectorySize() Thread.sleep(60000); @@ -218,11 +222,9 @@ public class ArbitraryDataCleanupManager extends Thread { } // Delete random data associated with name if we're over our storage limit for this name - // Use a threshold of 1 so that we only start deleting once the hard limit is reached - // This also allows some headroom between the regular threshold (90%) and the hard - // limit, to avoid data getting into a fetch/delete loop. + // Use the DELETION_THRESHOLD, for the same reasons as above for (String followedName : storageManager.followedNames()) { - if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, 1.0f)) { + if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) { this.storageLimitReachedForName(repository, followedName); } } @@ -281,7 +283,7 @@ public class ArbitraryDataCleanupManager extends Thread { // Now calculate the used/total storage again, as a safety precaution Long now = NTP.getTime(); ArbitraryDataStorageManager.getInstance().calculateDirectorySize(now); - if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailable(1.0f)) { + if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailable(DELETION_THRESHOLD)) { // We have space available, so don't delete anything return; } @@ -299,7 +301,7 @@ public class ArbitraryDataCleanupManager extends Thread { public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException { // We think that the storage limit has been reached for supplied name - but we should double check - if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, 1.0f)) { + if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, DELETION_THRESHOLD)) { // We have space available for this name, so don't delete anything return; } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index f8195e0d..e5e7d6e6 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -48,8 +48,13 @@ public class ArbitraryDataStorageManager extends Thread { /** Treat storage as full at 90% usage, to reduce risk of going over the limit. * This is necessary because we don't calculate total storage values before every write. - * It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit. */ - private static final double STORAGE_FULL_THRESHOLD = 0.9; // 90% + * It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit. + * This must be lower than DELETION_THRESHOLD. */ + private static final double STORAGE_FULL_THRESHOLD = 0.90f; // 90% + + /** Start deleting files once we reach 98% usage. + * This must be higher than STORAGE_FULL_THRESHOLD in order to avoid a fetch/delete loop. */ + public static final double DELETION_THRESHOLD = 0.98f; // 98% public ArbitraryDataStorageManager() { }