Qortal/qortal
1
0
Fork 0
mirror of https://github.com/Qortal/qortal.git synced 2025-04-14 23:35:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

removed name based arbitrary resource storage capacity limits and added arbitrary resource cache rebuild logging verbosity

Browse Source
This commit is contained in:
kennycud 2025-03-10 15:17:04 -07:00
parent 92fb52220a
commit 7f3c1d553f
4 changed files with 30 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java
View File

@ -15,13 +15,16 @@ import org.qortal.settings.Settings;
import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.utils.Base58;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
public class ArbitraryDataCacheManager extends Thread {
@ -34,6 +37,11 @@ public class ArbitraryDataCacheManager extends Thread {
/** Queue of arbitrary transactions that require cache updates */
private final List<ArbitraryTransactionData> updateQueue = Collections.synchronizedList(new ArrayList<>());
private static final NumberFormat FORMATTER = NumberFormat.getNumberInstance();
static {
FORMATTER.setGroupingUsed(true);
}
public static synchronized ArbitraryDataCacheManager getInstance() {
if (instance == null) {
@ -191,7 +199,12 @@ public class ArbitraryDataCacheManager extends Thread {
// Loop through all ARBITRARY transactions, and determine latest state
while (!Controller.isStopping()) {
LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1);
LOGGER.info(
"Fetching arbitrary transactions {} - {} / {} Total",
FORMATTER.format(offset),
FORMATTER.format(offset+batchSize-1),
FORMATTER.format(allArbitraryTransactionsInDescendingOrder.size())
);
List<ArbitraryTransactionData> transactionsToProcess
= allArbitraryTransactionsInDescendingOrder.stream()
@ -254,11 +267,25 @@ public class ArbitraryDataCacheManager extends Thread {
final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
int offset = 0;
List<ArbitraryTransactionData> allHostedTransactions
= ArbitraryDataStorageManager.getInstance()
.listAllHostedTransactions(repository, null, null);
// Loop through all ARBITRARY transactions, and determine latest state
while (!Controller.isStopping()) {
LOGGER.info("Fetching hosted transactions {} - {}", offset, offset+batchSize-1);
LOGGER.info(
"Fetching hosted transactions {} - {} / {} Total",
FORMATTER.format(offset),
FORMATTER.format(offset+batchSize-1),
FORMATTER.format(allHostedTransactions.size())
);
List<ArbitraryTransactionData> hostedTransactions
= allHostedTransactions.stream()
.skip(offset)
.limit(batchSize)
.collect(Collectors.toList());
List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, batchSize, offset);
if (hostedTransactions.isEmpty()) {
// Complete
break;

30
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
View File

@ -307,17 +307,6 @@ public class ArbitraryDataCleanupManager extends Thread {
this.storageLimitReached(repository);
}
// Delete random data associated with name if we're over our storage limit for this name
// Use the DELETION_THRESHOLD, for the same reasons as above
for (String followedName : ListUtils.followedNames()) {
if (isStopping) {
return;
}
if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) {
this.storageLimitReachedForName(repository, followedName);
}
}
} catch (DataException e) {
LOGGER.error("Repository issue when cleaning up arbitrary transaction data", e);
}
@ -396,25 +385,6 @@ public class ArbitraryDataCleanupManager extends Thread {
// FUTURE: consider reducing the expiry time of the reader cache
}
public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException {
// We think that the storage limit has been reached for supplied name - but we should double check
if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, DELETION_THRESHOLD)) {
// We have space available for this name, so don't delete anything
return;
}
// Delete a batch of random chunks associated with this name
// This reduces the chance of too many nodes deleting the same chunk
// when they reach their storage limit
Path dataPath = Paths.get(Settings.getInstance().getDataPath());
for (int i=0; i<CHUNK_DELETION_BATCH_SIZE; i++) {
if (isStopping) {
return;
}
this.deleteRandomFile(repository, dataPath.toFile(), name);
}
}
/**
* Iteratively walk through given directory and delete a single random file
*

52
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
View File

@ -170,13 +170,6 @@ public class ArbitraryDataStorageManager extends Thread {
return new ArbitraryDataExamination(false,"Don't fetch anything more if we're (nearly) out of space");
}
// Don't fetch anything if we're (nearly) out of space for this name
// Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
// avoid a fetch/delete loop
if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) {
return new ArbitraryDataExamination(false, "Don't fetch anything if we're (nearly) out of space for this name");
}
// Don't store data unless it's an allowed type (public/private)
if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
return new ArbitraryDataExamination(false, "Don't store data unless it's an allowed type (public/private)");
@ -484,51 +477,6 @@ public class ArbitraryDataStorageManager extends Thread {
return true;
}
public boolean isStorageSpaceAvailableForName(Repository repository, String name, double threshold) {
if (!this.isStorageSpaceAvailable(threshold)) {
// No storage space available at all, so no need to check this name
return false;
}
if (Settings.getInstance().getStoragePolicy() == StoragePolicy.ALL) {
// Using storage policy ALL, so don't limit anything per name
return true;
}
if (name == null) {
// This transaction doesn't have a name, so fall back to total space limitations
return true;
}
int followedNamesCount = ListUtils.followedNamesCount();
if (followedNamesCount == 0) {
// Not following any names, so we have space
return true;
}
long totalSizeForName = 0;
long maxStoragePerName = this.storageCapacityPerName(threshold);
// Fetch all hosted transactions
List<ArbitraryTransactionData> hostedTransactions = this.listAllHostedTransactions(repository, null, null);
for (ArbitraryTransactionData transactionData : hostedTransactions) {
String transactionName = transactionData.getName();
if (!Objects.equals(name, transactionName)) {
// Transaction relates to a different name
continue;
}
totalSizeForName += transactionData.getSize();
}
// Have we reached the limit for this name?
if (totalSizeForName > maxStoragePerName) {
return false;
}
return true;
}
public long storageCapacityPerName(double threshold) {
int followedNamesCount = ListUtils.followedNamesCount();
if (followedNamesCount == 0) {

50
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java
View File

@ -145,56 +145,6 @@ public class ArbitraryDataStorageCapacityTests extends Common {
}
}
@Test
public void testDeleteRandomFilesForName() throws DataException, IOException, InterruptedException, IllegalAccessException {
try (final Repository repository = RepositoryManager.getRepository()) {
String identifier = null; // Not used for this test
Service service = Service.ARBITRARY_DATA;
int chunkSize = 100;
int dataLength = 900; // Actual data length will be longer due to encryption
// Set originalCopyIndicatorFileEnabled to false, otherwise nothing will be deleted as it all originates from this node
FieldUtils.writeField(Settings.getInstance(), "originalCopyIndicatorFileEnabled", false, true);
// Alice hosts some data (with 10 chunks)
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String aliceName = "alice";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), aliceName, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
TransactionUtils.signAndMint(repository, transactionData, alice);
Path alicePath = ArbitraryUtils.generateRandomDataPath(dataLength);
ArbitraryDataFile aliceArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(alice.getPublicKey()), alicePath, aliceName, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize);
// Bob hosts some data too (also with 10 chunks)
PrivateKeyAccount bob = Common.getTestAccount(repository, "bob");
String bobName = "bob";
transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
TransactionUtils.signAndMint(repository, transactionData, bob);
Path bobPath = ArbitraryUtils.generateRandomDataPath(dataLength);
ArbitraryDataFile bobArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(bob.getPublicKey()), bobPath, bobName, identifier, ArbitraryTransactionData.Method.PUT, service, bob, chunkSize);
// All 20 chunks should exist
assertEquals(10, aliceArbitraryDataFile.chunkCount());
assertTrue(aliceArbitraryDataFile.allChunksExist());
assertEquals(10, bobArbitraryDataFile.chunkCount());
assertTrue(bobArbitraryDataFile.allChunksExist());
// Now pretend that Bob has reached his storage limit - this should delete random files
// Run it 10 times to remove the likelihood of the randomizer always picking Alice's files
for (int i=0; i<10; i++) {
ArbitraryDataCleanupManager.getInstance().storageLimitReachedForName(repository, bobName);
}
// Alice should still have all chunks
assertTrue(aliceArbitraryDataFile.allChunksExist());
// Bob should be missing some chunks
assertFalse(bobArbitraryDataFile.allChunksExist());
}
}
private void deleteListsDirectory() {
// Delete lists directory if exists
Path listsPath = Paths.get(Settings.getInstance().getListsPath());