From 7f3c1d553f3fb0d8c9a6906fb488014a0686d87a Mon Sep 17 00:00:00 2001 From: kennycud Date: Mon, 10 Mar 2025 15:17:04 -0700 Subject: [PATCH] removed name based arbitrary resource storage capacity limits and added arbitrary resource cache rebuild logging verbosity --- .../arbitrary/ArbitraryDataCacheManager.java | 33 ++++++++++-- .../ArbitraryDataCleanupManager.java | 30 ----------- .../ArbitraryDataStorageManager.java | 52 ------------------- .../ArbitraryDataStorageCapacityTests.java | 50 ------------------ 4 files changed, 30 insertions(+), 135 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java index 62d91109..cdc9bbaa 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java @@ -15,13 +15,16 @@ import org.qortal.settings.Settings; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.utils.Base58; +import java.text.NumberFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; public class ArbitraryDataCacheManager extends Thread { @@ -34,6 +37,11 @@ public class ArbitraryDataCacheManager extends Thread { /** Queue of arbitrary transactions that require cache updates */ private final List updateQueue = Collections.synchronizedList(new ArrayList<>()); + private static final NumberFormat FORMATTER = NumberFormat.getNumberInstance(); + + static { + FORMATTER.setGroupingUsed(true); + } public static synchronized ArbitraryDataCacheManager getInstance() { if (instance == null) { @@ -191,7 +199,12 @@ public class ArbitraryDataCacheManager extends Thread { // Loop through all ARBITRARY transactions, and determine latest state while (!Controller.isStopping()) { - LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1); + LOGGER.info( + "Fetching arbitrary transactions {} - {} / {} Total", + FORMATTER.format(offset), + FORMATTER.format(offset+batchSize-1), + FORMATTER.format(allArbitraryTransactionsInDescendingOrder.size()) + ); List transactionsToProcess = allArbitraryTransactionsInDescendingOrder.stream() @@ -254,11 +267,25 @@ public class ArbitraryDataCacheManager extends Thread { final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize(); int offset = 0; + List allHostedTransactions + = ArbitraryDataStorageManager.getInstance() + .listAllHostedTransactions(repository, null, null); + // Loop through all ARBITRARY transactions, and determine latest state while (!Controller.isStopping()) { - LOGGER.info("Fetching hosted transactions {} - {}", offset, offset+batchSize-1); + LOGGER.info( + "Fetching hosted transactions {} - {} / {} Total", + FORMATTER.format(offset), + FORMATTER.format(offset+batchSize-1), + FORMATTER.format(allHostedTransactions.size()) + ); + + List hostedTransactions + = allHostedTransactions.stream() + .skip(offset) + .limit(batchSize) + .collect(Collectors.toList()); - List hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, batchSize, offset); if (hostedTransactions.isEmpty()) { // Complete break; diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index bc92395d..ce4dd565 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -307,17 +307,6 @@ public class ArbitraryDataCleanupManager extends Thread { this.storageLimitReached(repository); } - // Delete random data associated with name if we're over our storage limit for this name - // Use the DELETION_THRESHOLD, for the same reasons as above - for (String followedName : ListUtils.followedNames()) { - if (isStopping) { - return; - } - if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) { - this.storageLimitReachedForName(repository, followedName); - } - } - } catch (DataException e) { LOGGER.error("Repository issue when cleaning up arbitrary transaction data", e); } @@ -396,25 +385,6 @@ public class ArbitraryDataCleanupManager extends Thread { // FUTURE: consider reducing the expiry time of the reader cache } - public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException { - // We think that the storage limit has been reached for supplied name - but we should double check - if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, DELETION_THRESHOLD)) { - // We have space available for this name, so don't delete anything - return; - } - - // Delete a batch of random chunks associated with this name - // This reduces the chance of too many nodes deleting the same chunk - // when they reach their storage limit - Path dataPath = Paths.get(Settings.getInstance().getDataPath()); - for (int i=0; i hostedTransactions = this.listAllHostedTransactions(repository, null, null); - for (ArbitraryTransactionData transactionData : hostedTransactions) { - String transactionName = transactionData.getName(); - if (!Objects.equals(name, transactionName)) { - // Transaction relates to a different name - continue; - } - - totalSizeForName += transactionData.getSize(); - } - - // Have we reached the limit for this name? - if (totalSizeForName > maxStoragePerName) { - return false; - } - - return true; - } - public long storageCapacityPerName(double threshold) { int followedNamesCount = ListUtils.followedNamesCount(); if (followedNamesCount == 0) { diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java index c05ceabf..33375b62 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java @@ -145,56 +145,6 @@ public class ArbitraryDataStorageCapacityTests extends Common { } } - @Test - public void testDeleteRandomFilesForName() throws DataException, IOException, InterruptedException, IllegalAccessException { - try (final Repository repository = RepositoryManager.getRepository()) { - String identifier = null; // Not used for this test - Service service = Service.ARBITRARY_DATA; - int chunkSize = 100; - int dataLength = 900; // Actual data length will be longer due to encryption - - // Set originalCopyIndicatorFileEnabled to false, otherwise nothing will be deleted as it all originates from this node - FieldUtils.writeField(Settings.getInstance(), "originalCopyIndicatorFileEnabled", false, true); - - // Alice hosts some data (with 10 chunks) - PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); - String aliceName = "alice"; - RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), aliceName, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); - TransactionUtils.signAndMint(repository, transactionData, alice); - Path alicePath = ArbitraryUtils.generateRandomDataPath(dataLength); - ArbitraryDataFile aliceArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(alice.getPublicKey()), alicePath, aliceName, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize); - - // Bob hosts some data too (also with 10 chunks) - PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); - String bobName = "bob"; - transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); - TransactionUtils.signAndMint(repository, transactionData, bob); - Path bobPath = ArbitraryUtils.generateRandomDataPath(dataLength); - ArbitraryDataFile bobArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(bob.getPublicKey()), bobPath, bobName, identifier, ArbitraryTransactionData.Method.PUT, service, bob, chunkSize); - - // All 20 chunks should exist - assertEquals(10, aliceArbitraryDataFile.chunkCount()); - assertTrue(aliceArbitraryDataFile.allChunksExist()); - assertEquals(10, bobArbitraryDataFile.chunkCount()); - assertTrue(bobArbitraryDataFile.allChunksExist()); - - // Now pretend that Bob has reached his storage limit - this should delete random files - // Run it 10 times to remove the likelihood of the randomizer always picking Alice's files - for (int i=0; i<10; i++) { - ArbitraryDataCleanupManager.getInstance().storageLimitReachedForName(repository, bobName); - } - - // Alice should still have all chunks - assertTrue(aliceArbitraryDataFile.allChunksExist()); - - // Bob should be missing some chunks - assertFalse(bobArbitraryDataFile.allChunksExist()); - - } - } - private void deleteListsDirectory() { // Delete lists directory if exists Path listsPath = Paths.get(Settings.getInstance().getListsPath());