removed name based arbitrary resource storage capacity limits and added arbitrary resource cache rebuild logging verbosity

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java

@@ -15,13 +15,16 @@ import org.qortal.settings.Settings;
 import org.qortal.transaction.ArbitraryTransaction;
 import org.qortal.utils.Base58;
 
+import java.text.NumberFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 public class ArbitraryDataCacheManager extends Thread {
@@ -34,6 +37,11 @@ public class ArbitraryDataCacheManager extends Thread {
     /** Queue of arbitrary transactions that require cache updates */
     private final List<ArbitraryTransactionData> updateQueue = Collections.synchronizedList(new ArrayList<>());
 
+    private static final NumberFormat FORMATTER = NumberFormat.getNumberInstance();
+
+    static {
+        FORMATTER.setGroupingUsed(true);
+    }
 
     public static synchronized ArbitraryDataCacheManager getInstance() {
         if (instance == null) {
@@ -191,7 +199,12 @@ public class ArbitraryDataCacheManager extends Thread {
 
             // Loop through all ARBITRARY transactions, and determine latest state
             while (!Controller.isStopping()) {
-                LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1);
+                LOGGER.info(
+                    "Fetching arbitrary transactions {} - {} / {} Total",
+                    FORMATTER.format(offset),
+                    FORMATTER.format(offset+batchSize-1),
+                    FORMATTER.format(allArbitraryTransactionsInDescendingOrder.size())
+                );
 
                 List<ArbitraryTransactionData> transactionsToProcess
                     = allArbitraryTransactionsInDescendingOrder.stream()
@@ -254,11 +267,25 @@ public class ArbitraryDataCacheManager extends Thread {
             final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
             int offset = 0;
 
+            List<ArbitraryTransactionData> allHostedTransactions
+                = ArbitraryDataStorageManager.getInstance()
+                    .listAllHostedTransactions(repository, null, null);
+
             // Loop through all ARBITRARY transactions, and determine latest state
             while (!Controller.isStopping()) {
-                LOGGER.info("Fetching hosted transactions {} - {}", offset, offset+batchSize-1);
+                LOGGER.info(
+                    "Fetching hosted transactions {} - {} / {} Total",
+                    FORMATTER.format(offset),
+                    FORMATTER.format(offset+batchSize-1),
+                    FORMATTER.format(allHostedTransactions.size())
+                );
+
+                List<ArbitraryTransactionData> hostedTransactions
+                    = allHostedTransactions.stream()
+                        .skip(offset)
+                        .limit(batchSize)
+                        .collect(Collectors.toList());
 
-                List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, batchSize, offset);
                 if (hostedTransactions.isEmpty()) {
                     // Complete
                     break;

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java

@@ -307,17 +307,6 @@ public class ArbitraryDataCleanupManager extends Thread {
 						this.storageLimitReached(repository);
 					}
 
-					// Delete random data associated with name if we're over our storage limit for this name
-					// Use the DELETION_THRESHOLD, for the same reasons as above
-					for (String followedName : ListUtils.followedNames()) {
-						if (isStopping) {
-							return;
-						}
-						if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) {
-							this.storageLimitReachedForName(repository, followedName);
-						}
-					}
-
 				} catch (DataException e) {
 					LOGGER.error("Repository issue when cleaning up arbitrary transaction data", e);
 				}
@@ -396,25 +385,6 @@ public class ArbitraryDataCleanupManager extends Thread {
 		// FUTURE: consider reducing the expiry time of the reader cache
 	}
 
-	public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException {
-		// We think that the storage limit has been reached for supplied name - but we should double check
-		if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, DELETION_THRESHOLD)) {
-			// We have space available for this name, so don't delete anything
-			return;
-		}
-
-		// Delete a batch of random chunks associated with this name
-		// This reduces the chance of too many nodes deleting the same chunk
-		// when they reach their storage limit
-		Path dataPath = Paths.get(Settings.getInstance().getDataPath());
-		for (int i=0; i<CHUNK_DELETION_BATCH_SIZE; i++) {
-			if (isStopping) {
-				return;
-			}
-			this.deleteRandomFile(repository, dataPath.toFile(), name);
-		}
-	}
-
 	/**
 	 * Iteratively walk through given directory and delete a single random file
 	 *

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java

@@ -170,13 +170,6 @@ public class ArbitraryDataStorageManager extends Thread {
             return new ArbitraryDataExamination(false,"Don't fetch anything more if we're (nearly) out of space");
         }
 
-        // Don't fetch anything if we're (nearly) out of space for this name
-        // Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
-        // avoid a fetch/delete loop
-        if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) {
-            return new ArbitraryDataExamination(false, "Don't fetch anything if we're (nearly) out of space for this name");
-        }
-
         // Don't store data unless it's an allowed type (public/private)
         if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
             return new ArbitraryDataExamination(false, "Don't store data unless it's an allowed type (public/private)");
@@ -484,51 +477,6 @@ public class ArbitraryDataStorageManager extends Thread {
         return true;
     }
 
-    public boolean isStorageSpaceAvailableForName(Repository repository, String name, double threshold) {
-        if (!this.isStorageSpaceAvailable(threshold)) {
-            // No storage space available at all, so no need to check this name
-            return false;
-        }
-
-        if (Settings.getInstance().getStoragePolicy() == StoragePolicy.ALL) {
-            // Using storage policy ALL, so don't limit anything per name
-            return true;
-        }
-
-        if (name == null) {
-            // This transaction doesn't have a name, so fall back to total space limitations
-            return true;
-        }
-
-        int followedNamesCount = ListUtils.followedNamesCount();
-        if (followedNamesCount == 0) {
-            // Not following any names, so we have space
-            return true;
-        }
-
-        long totalSizeForName = 0;
-        long maxStoragePerName = this.storageCapacityPerName(threshold);
-
-        // Fetch all hosted transactions
-        List<ArbitraryTransactionData> hostedTransactions = this.listAllHostedTransactions(repository, null, null);
-        for (ArbitraryTransactionData transactionData : hostedTransactions) {
-            String transactionName = transactionData.getName();
-            if (!Objects.equals(name, transactionName)) {
-                // Transaction relates to a different name
-                continue;
-            }
-
-            totalSizeForName += transactionData.getSize();
-        }
-
-        // Have we reached the limit for this name?
-        if (totalSizeForName > maxStoragePerName) {
-            return false;
-        }
-
-        return true;
-    }
-
     public long storageCapacityPerName(double threshold) {
         int followedNamesCount = ListUtils.followedNamesCount();
         if (followedNamesCount == 0) {

src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java

@@ -145,56 +145,6 @@ public class ArbitraryDataStorageCapacityTests extends Common {
         }
     }
 
-    @Test
-    public void testDeleteRandomFilesForName() throws DataException, IOException, InterruptedException, IllegalAccessException {
-        try (final Repository repository = RepositoryManager.getRepository()) {
-            String identifier = null; // Not used for this test
-            Service service = Service.ARBITRARY_DATA;
-            int chunkSize = 100;
-            int dataLength = 900; // Actual data length will be longer due to encryption
-
-            // Set originalCopyIndicatorFileEnabled to false, otherwise nothing will be deleted as it all originates from this node
-            FieldUtils.writeField(Settings.getInstance(), "originalCopyIndicatorFileEnabled", false, true);
-
-            // Alice hosts some data (with 10 chunks)
-            PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
-            String aliceName = "alice";
-            RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), aliceName, "");
-            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
-            TransactionUtils.signAndMint(repository, transactionData, alice);
-            Path alicePath = ArbitraryUtils.generateRandomDataPath(dataLength);
-            ArbitraryDataFile aliceArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(alice.getPublicKey()), alicePath, aliceName, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize);
-
-            // Bob hosts some data too (also with 10 chunks)
-            PrivateKeyAccount bob = Common.getTestAccount(repository, "bob");
-            String bobName = "bob";
-            transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, "");
-            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
-            TransactionUtils.signAndMint(repository, transactionData, bob);
-            Path bobPath = ArbitraryUtils.generateRandomDataPath(dataLength);
-            ArbitraryDataFile bobArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(bob.getPublicKey()), bobPath, bobName, identifier, ArbitraryTransactionData.Method.PUT, service, bob, chunkSize);
-
-            // All 20 chunks should exist
-            assertEquals(10, aliceArbitraryDataFile.chunkCount());
-            assertTrue(aliceArbitraryDataFile.allChunksExist());
-            assertEquals(10, bobArbitraryDataFile.chunkCount());
-            assertTrue(bobArbitraryDataFile.allChunksExist());
-
-            // Now pretend that Bob has reached his storage limit - this should delete random files
-            // Run it 10 times to remove the likelihood of the randomizer always picking Alice's files
-            for (int i=0; i<10; i++) {
-                ArbitraryDataCleanupManager.getInstance().storageLimitReachedForName(repository, bobName);
-            }
-
-            // Alice should still have all chunks
-            assertTrue(aliceArbitraryDataFile.allChunksExist());
-
-            // Bob should be missing some chunks
-            assertFalse(bobArbitraryDataFile.allChunksExist());
-
-        }
-    }
-
     private void deleteListsDirectory() {
         // Delete lists directory if exists
         Path listsPath = Paths.get(Settings.getInstance().getListsPath());