diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 5264e424..64380cbc 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -609,7 +609,7 @@ public class ArbitraryResource { return hostedTransactions; - } catch (DataException | IOException e) { + } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } } @@ -657,7 +657,7 @@ public class ArbitraryResource { return resources; - } catch (DataException | IOException e) { + } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 610edf67..d3b60531 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -190,7 +190,7 @@ public class ArbitraryDataManager extends Thread { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData(); // Skip transactions that we don't need to proactively store data for - if (!storageManager.shouldPreFetchData(arbitraryTransactionData)) { + if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) { iterator.remove(); continue; } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index de138b16..1b66c0d3 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -20,6 +20,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; public class ArbitraryDataStorageManager extends Thread { @@ -43,12 +44,12 @@ public class ArbitraryDataStorageManager extends Thread { private List hostedTransactions; - private static long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes + private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes /** Treat storage as full at 90% usage, to reduce risk of going over the limit. * This is necessary because we don't calculate total storage values before every write. * It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit. */ - private static double STORAGE_FULL_THRESHOLD = 0.9; // 90% + private static final double STORAGE_FULL_THRESHOLD = 0.9; // 90% public ArbitraryDataStorageManager() { } @@ -133,7 +134,7 @@ public class ArbitraryDataStorageManager extends Thread { * @param arbitraryTransactionData - the transaction * @return boolean - whether to prefetch or not */ - public boolean shouldPreFetchData(ArbitraryTransactionData arbitraryTransactionData) { + public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { String name = arbitraryTransactionData.getName(); // Don't fetch anything more if we're (nearly) out of space @@ -143,6 +144,13 @@ public class ArbitraryDataStorageManager extends Thread { return false; } + // Don't fetch anything if we're (nearly) out of space for this name + // Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to + // avoid a fetch/delete loop + if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) { + return false; + } + // Don't store data unless it's an allowed type (public/private) if (!this.isDataTypeAllowed(arbitraryTransactionData)) { return false; @@ -217,10 +225,14 @@ public class ArbitraryDataStorageManager extends Thread { return ResourceListManager.getInstance().listContains("followed", "names", name, false); } + private int followedNamesCount() { + return ResourceListManager.getInstance().getItemCountForList("followed", "names"); + } + // Hosted data - public List listAllHostedTransactions(Repository repository) throws IOException { + public List listAllHostedTransactions(Repository repository) { // Load from cache if we can, to avoid disk reads if (this.hostedTransactions != null) { return this.hostedTransactions; @@ -233,12 +245,18 @@ public class ArbitraryDataStorageManager extends Thread { // Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length // Also exclude the _temp and _misc paths if present - List allPaths = Files.walk(dataPath, 3) - .filter(Files::isDirectory) - .filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString()) - && !path.toString().contains("_misc") - && path.getFileName().toString().length() > 32) - .collect(Collectors.toList()); + List allPaths = new ArrayList<>(); + try { + allPaths = Files.walk(dataPath, 3) + .filter(Files::isDirectory) + .filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString()) + && !path.toString().contains("_misc") + && path.getFileName().toString().length() > 32) + .collect(Collectors.toList()); + } + catch (IOException e) { + LOGGER.info("Unable to walk through hosted data: {}", e.getMessage()); + } // Loop through each path and attempt to match it to a signature for (Path path : allPaths) { @@ -277,7 +295,7 @@ public class ArbitraryDataStorageManager extends Thread { /** * Rate limit to reduce IO load */ - private boolean shouldCalculateDirectorySize(Long now) { + public boolean shouldCalculateDirectorySize(Long now) { if (now == null) { return false; } @@ -368,7 +386,71 @@ public class ArbitraryDataStorageManager extends Thread { return true; } + public boolean isStorageSpaceAvailableForName(Repository repository, String name, double threshold) { + if (!this.isStorageSpaceAvailable(threshold)) { + // No storage space available at all, so no need to check this name + return false; + } + + if (name == null) { + // This transaction doesn't have a name, so fall back to total space limitations + return true; + } + + int followedNamesCount = this.followedNamesCount(); + if (followedNamesCount == 0) { + // Not following any names, so we have space + return true; + } + + long totalSizeForName = 0; + long maxStoragePerName = this.storageCapacityPerName(threshold); + + // Fetch all hosted transactions + List hostedTransactions = this.listAllHostedTransactions(repository); + for (ArbitraryTransactionData transactionData : hostedTransactions) { + String transactionName = transactionData.getName(); + if (!Objects.equals(name, transactionName)) { + // Transaction relates to a different name + continue; + } + + totalSizeForName += transactionData.getSize(); + } + + // Have we reached the limit for this name? + if (totalSizeForName > maxStoragePerName) { + return false; + } + + return true; + } + + public long storageCapacityPerName(double threshold) { + int followedNamesCount = this.followedNamesCount(); + if (followedNamesCount == 0) { + // Not following any names, so we have the total space available + return this.getStorageCapacityIncludingThreshold(threshold); + } + + double maxStorageCapacity = (double)this.storageCapacity * threshold; + long maxStoragePerName = (long)(maxStorageCapacity / (double)followedNamesCount); + + return maxStoragePerName; + } + public boolean isStorageCapacityCalculated() { return (this.storageCapacity != null); } + + public Long getStorageCapacity() { + return this.storageCapacity; + } + + public Long getStorageCapacityIncludingThreshold(double threshold) { + if (this.storageCapacity == null) { + return null; + } + return (long)(this.storageCapacity * threshold); + } } diff --git a/src/main/java/org/qortal/list/ResourceListManager.java b/src/main/java/org/qortal/list/ResourceListManager.java index 921105e9..c71dfdab 100644 --- a/src/main/java/org/qortal/list/ResourceListManager.java +++ b/src/main/java/org/qortal/list/ResourceListManager.java @@ -133,4 +133,12 @@ public class ResourceListManager { return list.getList(); } + public int getItemCountForList(String category, String resourceName) { + ResourceList list = this.getList(category, resourceName); + if (list == null) { + return 0; + } + return list.getList().size(); + } + } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java new file mode 100644 index 00000000..f40da628 --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java @@ -0,0 +1,134 @@ +package org.qortal.test.arbitrary; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; +import org.qortal.list.ResourceListManager; +import org.qortal.repository.DataException; +import org.qortal.settings.Settings; +import org.qortal.test.common.Common; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.Assert.*; + +public class ArbitraryDataStorageCapacityTests extends Common { + + @Before + public void beforeTest() throws DataException, InterruptedException { + Common.useDefaultSettings(); + this.deleteDataDirectories(); + this.deleteListsDirectory(); + } + + @After + public void afterTest() throws DataException { + this.deleteDataDirectories(); + this.deleteListsDirectory(); + ArbitraryDataStorageManager.getInstance().shutdown(); + } + + + @Test + public void testCalculateTotalStorageCapacity() { + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + double storageFullThreshold = 0.9; // 90% + Long now = NTP.getTime(); + assertNotNull("NTP time must be synced", now); + long expectedTotalStorageCapacity = Settings.getInstance().getMaxStorageCapacity(); + + // Capacity isn't initially calculated + assertNull(storageManager.getStorageCapacity()); + assertEquals(0L, storageManager.getTotalDirectorySize()); + assertFalse(storageManager.isStorageCapacityCalculated()); + + // We need to calculate the directory size because we haven't yet + assertTrue(storageManager.shouldCalculateDirectorySize(now)); + storageManager.calculateDirectorySize(now); + assertTrue(storageManager.isStorageCapacityCalculated()); + + // Storage capacity should equal the value specified in settings + assertNotNull(storageManager.getStorageCapacity()); + assertEquals(expectedTotalStorageCapacity, storageManager.getStorageCapacity().longValue()); + + // We shouldn't calculate storage capacity again so soon + now += 9 * 60 * 1000L; + assertFalse(storageManager.shouldCalculateDirectorySize(now)); + + // ... but after 10 minutes we should recalculate + now += 1 * 60 * 1000L + 1L; + assertTrue(storageManager.shouldCalculateDirectorySize(now)); + } + + @Test + public void testCalculateStorageCapacityPerName() { + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + ResourceListManager resourceListManager = ResourceListManager.getInstance(); + double storageFullThreshold = 0.9; // 90% + Long now = NTP.getTime(); + assertNotNull("NTP time must be synced", now); + + // Capacity isn't initially calculated + assertNull(storageManager.getStorageCapacity()); + assertEquals(0L, storageManager.getTotalDirectorySize()); + assertFalse(storageManager.isStorageCapacityCalculated()); + + // We need to calculate the total directory size because we haven't yet + assertTrue(storageManager.shouldCalculateDirectorySize(now)); + storageManager.calculateDirectorySize(now); + assertTrue(storageManager.isStorageCapacityCalculated()); + + // Storage capacity should initially equal the total + assertEquals(0, resourceListManager.getItemCountForList("followed", "names")); + long totalStorageCapacity = storageManager.getStorageCapacityIncludingThreshold(storageFullThreshold); + assertEquals(totalStorageCapacity, storageManager.storageCapacityPerName(storageFullThreshold)); + + // Follow some names + assertTrue(resourceListManager.addToList("followed", "names", "Test1", false)); + assertTrue(resourceListManager.addToList("followed", "names", "Test2", false)); + assertTrue(resourceListManager.addToList("followed", "names", "Test3", false)); + assertTrue(resourceListManager.addToList("followed", "names", "Test4", false)); + + // Ensure the followed name count is correct + assertEquals(4, resourceListManager.getItemCountForList("followed", "names")); + + // Storage space per name should be the total storage capacity divided by the number of names + long expectedStorageCapacityPerName = (long)(totalStorageCapacity / 4.0f); + assertEquals(expectedStorageCapacityPerName, storageManager.storageCapacityPerName(storageFullThreshold)); + } + + + private void deleteDataDirectories() { + // Delete data directory if exists + Path dataPath = Paths.get(Settings.getInstance().getDataPath()); + try { + FileUtils.deleteDirectory(dataPath.toFile()); + } catch (IOException e) { + + } + + // Delete temp data directory if exists + Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath()); + try { + FileUtils.deleteDirectory(tempDataPath.toFile()); + } catch (IOException e) { + + } + } + + private void deleteListsDirectory() { + // Delete lists directory if exists + Path listsPath = Paths.get(Settings.getInstance().getListsPath()); + try { + FileUtils.deleteDirectory(listsPath.toFile()); + } catch (IOException e) { + + } + } + +} diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java index 317a015a..28edaa66 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java @@ -34,6 +34,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { @Before public void beforeTest() throws DataException, InterruptedException { Common.useDefaultSettings(); + this.deleteDataDirectories(); this.deleteListsDirectory(); ArbitraryDataStorageManager.getInstance().start(); @@ -45,6 +46,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { @After public void afterTest() throws DataException { + this.deleteDataDirectories(); this.deleteListsDirectory(); ArbitraryDataStorageManager.getInstance().shutdown(); } @@ -68,14 +70,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.FOLLOWED_AND_VIEWED, Settings.getInstance().getStoragePolicy()); assertTrue(storageManager.canStoreData(transactionData)); - assertTrue(storageManager.shouldPreFetchData(transactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, transactionData)); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false)); // We should store but not pre-fetch data for this transaction assertTrue(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(transactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); } } @@ -101,14 +103,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy()); assertTrue(storageManager.canStoreData(transactionData)); - assertTrue(storageManager.shouldPreFetchData(transactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, transactionData)); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false)); // We shouldn't store or pre-fetch data for this transaction assertFalse(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(transactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); } } @@ -134,14 +136,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store but not pre-fetch data for this transaction assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy()); assertTrue(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(transactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false)); // We should store but not pre-fetch data for this transaction assertTrue(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(transactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); } } @@ -167,14 +169,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy()); assertTrue(storageManager.canStoreData(transactionData)); - assertTrue(storageManager.shouldPreFetchData(transactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, transactionData)); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false)); // We should store and pre-fetch data for this transaction assertTrue(storageManager.canStoreData(transactionData)); - assertTrue(storageManager.shouldPreFetchData(transactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, transactionData)); } } @@ -200,14 +202,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We shouldn't store or pre-fetch data for this transaction assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy()); assertFalse(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(transactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false)); // We shouldn't store or pre-fetch data for this transaction assertFalse(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(transactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); } } @@ -223,7 +225,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store but not pre-fetch data for this transaction assertTrue(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(transactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); } } @@ -240,6 +242,24 @@ public class ArbitraryDataStoragePolicyTests extends Common { return transactionData; } + private void deleteDataDirectories() { + // Delete data directory if exists + Path dataPath = Paths.get(Settings.getInstance().getDataPath()); + try { + FileUtils.deleteDirectory(dataPath.toFile()); + } catch (IOException e) { + + } + + // Delete temp data directory if exists + Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath()); + try { + FileUtils.deleteDirectory(tempDataPath.toFile()); + } catch (IOException e) { + + } + } + private void deleteListsDirectory() { // Delete lists directory if exists Path listsPath = Paths.get(Settings.getInstance().getListsPath()); diff --git a/src/test/resources/test-settings-v2.json b/src/test/resources/test-settings-v2.json index f712245b..d9f6cb0c 100644 --- a/src/test/resources/test-settings-v2.json +++ b/src/test/resources/test-settings-v2.json @@ -15,5 +15,6 @@ "tempDataPath": "data-test/_temp", "listsPath": "lists-test", "storagePolicy": "FOLLOWED_AND_VIEWED", + "maxStorageCapacity": 104857600, "localAuthBypassEnabled": true }