Qortal/qortal
1
0
Fork 0
mirror of https://github.com/Qortal/qortal.git synced 2025-04-16 16:15:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'kenny/master' into feature/search-keywords

Browse Source
This commit is contained in:
PhilReact 2025-03-05 21:14:47 +02:00
commit 1d79df078e
12 changed files with 395 additions and 295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java
View File

@ -2,7 +2,6 @@ package org.qortal.controller.arbitrary;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.controller.Controller; import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData;
@ -14,12 +13,16 @@ import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager; import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings; import org.qortal.settings.Settings;
import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.Transaction;
import org.qortal.utils.Base58; import org.qortal.utils.Base58;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
public class ArbitraryDataCacheManager extends Thread { public class ArbitraryDataCacheManager extends Thread {
@ -86,47 +89,10 @@ public class ArbitraryDataCacheManager extends Thread {
// Update arbitrary resource caches // Update arbitrary resource caches
try { try {
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
transactionData.getIdentifier(),
transactionData.getName(),
transactionData.getService().name(),
"updating resource cache, queue",
transactionData.getTimestamp(),
transactionData.getTimestamp()
)
);
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceCache(repository); arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, new HashSet<>(0), new HashMap<>(0));
arbitraryTransaction.updateArbitraryMetadataCache(repository);
repository.saveChanges(); repository.saveChanges();
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
transactionData.getIdentifier(),
transactionData.getName(),
transactionData.getService().name(),
"updated resource cache",
transactionData.getTimestamp(),
transactionData.getTimestamp()
)
);
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
transactionData.getIdentifier(),
transactionData.getName(),
transactionData.getService().name(),
"updating resource status, queue",
transactionData.getTimestamp(),
transactionData.getTimestamp()
)
);
// Update status as separate commit, as this is more prone to failure // Update status as separate commit, as this is more prone to failure
arbitraryTransaction.updateArbitraryResourceStatus(repository); arbitraryTransaction.updateArbitraryResourceStatus(repository);
repository.saveChanges(); repository.saveChanges();
@ -137,7 +103,7 @@ public class ArbitraryDataCacheManager extends Thread {
transactionData.getIdentifier(), transactionData.getIdentifier(),
transactionData.getName(), transactionData.getName(),
transactionData.getService().name(), transactionData.getService().name(),
"updated resource status", "updated resource cache and status, queue",
transactionData.getTimestamp(), transactionData.getTimestamp(),
transactionData.getTimestamp() transactionData.getTimestamp()
) )
@ -201,64 +167,61 @@ public class ArbitraryDataCacheManager extends Thread {
LOGGER.info("Building arbitrary resources cache..."); LOGGER.info("Building arbitrary resources cache...");
SplashFrame.getInstance().updateStatus("Building QDN cache - please wait..."); SplashFrame.getInstance().updateStatus("Building QDN cache - please wait...");
final int batchSize = 100; final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
int offset = 0; int offset = 0;
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository().getLatestArbitraryTransactions();
LOGGER.info("arbitrary transactions: count = " + allArbitraryTransactionsInDescendingOrder.size());
List<ArbitraryResourceData> resources = repository.getArbitraryRepository().getArbitraryResources(null, null, true);
Map<ArbitraryTransactionDataHashWrapper, ArbitraryResourceData> resourceByWrapper = new HashMap<>(resources.size());
for( ArbitraryResourceData resource : resources ) {
resourceByWrapper.put(
new ArbitraryTransactionDataHashWrapper(resource.service.value, resource.name, resource.identifier),
resource
);
}
LOGGER.info("arbitrary resources: count = " + resourceByWrapper.size());
Set<ArbitraryTransactionDataHashWrapper> latestTransactionsWrapped = new HashSet<>(allArbitraryTransactionsInDescendingOrder.size());
// Loop through all ARBITRARY transactions, and determine latest state // Loop through all ARBITRARY transactions, and determine latest state
while (!Controller.isStopping()) { while (!Controller.isStopping()) {
LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1); LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1);
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false); List<ArbitraryTransactionData> transactionsToProcess
if (signatures.isEmpty()) { = allArbitraryTransactionsInDescendingOrder.stream()
.skip(offset)
.limit(batchSize)
.collect(Collectors.toList());
if (transactionsToProcess.isEmpty()) {
// Complete // Complete
break; break;
} }
// Expand signatures to transactions try {
for (byte[] signature : signatures) { for( ArbitraryTransactionData transactionData : transactionsToProcess) {
try {
ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository
.getTransactionRepository().fromSignature(signature);
if (transactionData.getService() == null) { if (transactionData.getService() == null) {
// Unsupported service - ignore this resource // Unsupported service - ignore this resource
continue; continue;
} }
EventBus.INSTANCE.notify( latestTransactionsWrapped.add(new ArbitraryTransactionDataHashWrapper(transactionData));
new DataMonitorEvent(
System.currentTimeMillis(),
transactionData.getIdentifier(),
transactionData.getName(),
transactionData.getService().name(),
"updating resource cache, build",
transactionData.getTimestamp(),
transactionData.getTimestamp()
)
);
// Update arbitrary resource caches // Update arbitrary resource caches
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceCache(repository); arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, latestTransactionsWrapped, resourceByWrapper);
arbitraryTransaction.updateArbitraryMetadataCache(repository);
repository.saveChanges();
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
transactionData.getIdentifier(),
transactionData.getName(),
transactionData.getService().name(),
"updated resource cache",
transactionData.getTimestamp(),
transactionData.getTimestamp()
)
);
} catch (DataException e) {
repository.discardChanges();
LOGGER.error(e.getMessage(), e);
} }
repository.saveChanges();
} catch (DataException e) {
repository.discardChanges();
LOGGER.error(e.getMessage(), e);
} }
offset += batchSize; offset += batchSize;
} }
@ -288,7 +251,7 @@ public class ArbitraryDataCacheManager extends Thread {
LOGGER.info("Refreshing arbitrary resource statuses for locally hosted transactions..."); LOGGER.info("Refreshing arbitrary resource statuses for locally hosted transactions...");
SplashFrame.getInstance().updateStatus("Refreshing statuses - please wait..."); SplashFrame.getInstance().updateStatus("Refreshing statuses - please wait...");
final int batchSize = 100; final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
int offset = 0; int offset = 0;
// Loop through all ARBITRARY transactions, and determine latest state // Loop through all ARBITRARY transactions, and determine latest state
@ -301,45 +264,21 @@ public class ArbitraryDataCacheManager extends Thread {
break; break;
} }
// Loop through hosted transactions try {
for (ArbitraryTransactionData transactionData : hostedTransactions) { // Loop through hosted transactions
for (ArbitraryTransactionData transactionData : hostedTransactions) {
try {
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
transactionData.getIdentifier(),
transactionData.getName(),
transactionData.getService().name(),
"updating resource status",
transactionData.getTimestamp(),
transactionData.getTimestamp()
)
);
// Determine status and update cache // Determine status and update cache
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceStatus(repository); arbitraryTransaction.updateArbitraryResourceStatus(repository);
repository.saveChanges();
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
transactionData.getIdentifier(),
transactionData.getName(),
transactionData.getService().name(),
"updated resource status",
transactionData.getTimestamp(),
transactionData.getTimestamp()
)
);
} catch (DataException e) {
repository.discardChanges();
LOGGER.error(e.getMessage(), e);
} }
repository.saveChanges();
} catch (DataException e) {
repository.discardChanges();
LOGGER.error(e.getMessage(), e);
} }
offset += batchSize; offset += batchSize;
} }

150
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
View File

@ -2,7 +2,6 @@ package org.qortal.controller.arbitrary;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.TransactionData;
import org.qortal.event.DataMonitorEvent; import org.qortal.event.DataMonitorEvent;
@ -23,9 +22,12 @@ import java.nio.file.Paths;
import java.security.SecureRandom; import java.security.SecureRandom;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD; import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD;
@ -80,6 +82,19 @@ public class ArbitraryDataCleanupManager extends Thread {
final int limit = 100; final int limit = 100;
int offset = 0; int offset = 0;
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
try (final Repository repository = RepositoryManager.getRepository()) {
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactions();
} catch( Exception e) {
LOGGER.error(e.getMessage(), e);
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
}
Set<ArbitraryTransactionData> processedTransactions = new HashSet<>();
try { try {
while (!isStopping) { while (!isStopping) {
Thread.sleep(30000); Thread.sleep(30000);
@ -110,27 +125,31 @@ public class ArbitraryDataCleanupManager extends Thread {
// Any arbitrary transactions we want to fetch data for? // Any arbitrary transactions we want to fetch data for?
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true); List<ArbitraryTransactionData> transactions = allArbitraryTransactionsInDescendingOrder.stream().skip(offset).limit(limit).collect(Collectors.toList());
// LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
if (isStopping) { if (isStopping) {
return; return;
} }
if (signatures == null || signatures.isEmpty()) { if (transactions == null || transactions.isEmpty()) {
offset = 0; offset = 0;
continue; allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactions();
transactions = allArbitraryTransactionsInDescendingOrder.stream().limit(limit).collect(Collectors.toList());
processedTransactions.clear();
} }
offset += limit; offset += limit;
now = NTP.getTime(); now = NTP.getTime();
// Loop through the signatures in this batch // Loop through the signatures in this batch
for (int i=0; i<signatures.size(); i++) { for (int i=0; i<transactions.size(); i++) {
if (isStopping) { if (isStopping) {
return; return;
} }
byte[] signature = signatures.get(i); ArbitraryTransactionData arbitraryTransactionData = transactions.get(i);
if (signature == null) { if (arbitraryTransactionData == null) {
continue; continue;
} }
@ -139,9 +158,7 @@ public class ArbitraryDataCleanupManager extends Thread {
Thread.sleep(5000); Thread.sleep(5000);
} }
// Fetch the transaction data if (arbitraryTransactionData.getService() == null) {
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
if (arbitraryTransactionData == null || arbitraryTransactionData.getService() == null) {
continue; continue;
} }
@ -150,6 +167,8 @@ public class ArbitraryDataCleanupManager extends Thread {
continue; continue;
} }
boolean mostRecentTransaction = processedTransactions.add(arbitraryTransactionData);
// Check if we have the complete file // Check if we have the complete file
boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData); boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData);
@ -186,28 +205,38 @@ public class ArbitraryDataCleanupManager extends Thread {
} }
// Check to see if we have had a more recent PUT // Check to see if we have had a more recent PUT
Optional<ArbitraryTransactionData> moreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData); if (!mostRecentTransaction) {
boolean hasMoreRecentPutTransaction = moreRecentPutTransaction.isPresent();
if (hasMoreRecentPutTransaction) {
// There is a more recent PUT transaction than the one we are currently processing. // There is a more recent PUT transaction than the one we are currently processing.
// When a PUT is issued, it replaces any layers that would have been there before. // When a PUT is issued, it replaces any layers that would have been there before.
// Therefore any data relating to this older transaction is no longer needed. // Therefore any data relating to this older transaction is no longer needed.
LOGGER.info(String.format("Newer PUT found for %s %s since transaction %s. " + LOGGER.info(String.format("Newer PUT found for %s %s since transaction %s. " +
"Deleting all files associated with the earlier transaction.", arbitraryTransactionData.getService(), "Deleting all files associated with the earlier transaction.", arbitraryTransactionData.getService(),
arbitraryTransactionData.getName(), Base58.encode(signature))); arbitraryTransactionData.getName(), Base58.encode(arbitraryTransactionData.getSignature())));
ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData); ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
EventBus.INSTANCE.notify(
new DataMonitorEvent( Optional<ArbitraryTransactionData> moreRecentPutTransaction
System.currentTimeMillis(), = processedTransactions.stream()
arbitraryTransactionData.getIdentifier(), .filter(data -> data.equals(arbitraryTransactionData))
arbitraryTransactionData.getName(), .findAny();
arbitraryTransactionData.getService().name(),
"deleting data due to replacement", if( moreRecentPutTransaction.isPresent() ) {
arbitraryTransactionData.getTimestamp(), EventBus.INSTANCE.notify(
moreRecentPutTransaction.get().getTimestamp() new DataMonitorEvent(
) System.currentTimeMillis(),
); arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"deleting data due to replacement",
arbitraryTransactionData.getTimestamp(),
moreRecentPutTransaction.get().getTimestamp()
)
);
}
else {
LOGGER.warn("Something went wrong with the most recent put transaction determination!");
}
continue; continue;
} }
@ -226,18 +255,21 @@ public class ArbitraryDataCleanupManager extends Thread {
LOGGER.debug(String.format("Transaction %s has complete file and all chunks", LOGGER.debug(String.format("Transaction %s has complete file and all chunks",
Base58.encode(arbitraryTransactionData.getSignature()))); Base58.encode(arbitraryTransactionData.getSignature())));
ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT); boolean wasDeleted = ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
EventBus.INSTANCE.notify(
new DataMonitorEvent( if( wasDeleted ) {
System.currentTimeMillis(), EventBus.INSTANCE.notify(
arbitraryTransactionData.getIdentifier(), new DataMonitorEvent(
arbitraryTransactionData.getName(), System.currentTimeMillis(),
arbitraryTransactionData.getService().name(), arbitraryTransactionData.getIdentifier(),
"deleting data due to age", arbitraryTransactionData.getName(),
arbitraryTransactionData.getTimestamp(), arbitraryTransactionData.getService().name(),
arbitraryTransactionData.getTimestamp() "deleting file, retaining chunks",
) arbitraryTransactionData.getTimestamp(),
); arbitraryTransactionData.getTimestamp()
)
);
}
continue; continue;
} }
@ -452,18 +484,6 @@ public class ArbitraryDataCleanupManager extends Thread {
// Relates to a different name - don't delete it // Relates to a different name - don't delete it
return false; return false;
} }
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"randomly selected for deletion",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
} }
} catch (DataException e) { } catch (DataException e) {
@ -473,6 +493,7 @@ public class ArbitraryDataCleanupManager extends Thread {
} }
LOGGER.info("Deleting random file {} because we have reached max storage capacity...", randomItem.toString()); LOGGER.info("Deleting random file {} because we have reached max storage capacity...", randomItem.toString());
fireRandomItemDeletionNotification(randomItem, repository, "Deleting random file, because we have reached max storage capacity");
boolean success = randomItem.delete(); boolean success = randomItem.delete();
if (success) { if (success) {
try { try {
@ -487,6 +508,35 @@ public class ArbitraryDataCleanupManager extends Thread {
return false; return false;
} }
private void fireRandomItemDeletionNotification(File randomItem, Repository repository, String reason) {
try {
Path parentFileNamePath = randomItem.toPath().toAbsolutePath().getParent().getFileName();
if (parentFileNamePath != null) {
String signature58 = parentFileNamePath.toString();
byte[] signature = Base58.decode(signature58);
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData != null && transactionData.getType() == Transaction.TransactionType.ARBITRARY) {
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
reason,
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
}
}
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
private void cleanupTempDirectory(String folder, long now, long minAge) { private void cleanupTempDirectory(String folder, long now, long minAge) {
String baseDir = Settings.getInstance().getTempDataPath(); String baseDir = Settings.getInstance().getTempDataPath();
Path tempDir = Paths.get(baseDir, folder); Path tempDir = Paths.get(baseDir, folder);

21
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataExamination.java Normal file
View File

@ -0,0 +1,21 @@
package org.qortal.controller.arbitrary;
public class ArbitraryDataExamination {
private boolean pass;
private String notes;
public ArbitraryDataExamination(boolean pass, String notes) {
this.pass = pass;
this.notes = notes;
}
public boolean isPass() {
return pass;
}
public String getNotes() {
return notes;
}
}

53
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@ -198,13 +198,35 @@ public class ArbitraryDataManager extends Thread {
final int limit = 100; final int limit = 100;
int offset = 0; int offset = 0;
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
try (final Repository repository = RepositoryManager.getRepository()) {
if( name == null ) {
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactions();
}
else {
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactionsByName(name);
}
} catch( Exception e) {
LOGGER.error(e.getMessage(), e);
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
}
// collect processed transactions in a set to ensure outdated data transactions do not get fetched
Set<ArbitraryTransactionDataHashWrapper> processedTransactions = new HashSet<>();
while (!isStopping) { while (!isStopping) {
Thread.sleep(1000L); Thread.sleep(1000L);
// Any arbitrary transactions we want to fetch data for? // Any arbitrary transactions we want to fetch data for?
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, name, null, ConfirmationStatus.BOTH, limit, offset, true); List<byte[]> signatures = processTransactionsForSignatures(limit, offset, allArbitraryTransactionsInDescendingOrder, processedTransactions);
// LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
if (signatures == null || signatures.isEmpty()) { if (signatures == null || signatures.isEmpty()) {
offset = 0; offset = 0;
break; break;
@ -226,7 +248,8 @@ public class ArbitraryDataManager extends Thread {
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData(); ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
// Skip transactions that we don't need to proactively store data for // Skip transactions that we don't need to proactively store data for
if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) { ArbitraryDataExamination arbitraryDataExamination = storageManager.shouldPreFetchData(repository, arbitraryTransactionData);
if (!arbitraryDataExamination.isPass()) {
iterator.remove(); iterator.remove();
EventBus.INSTANCE.notify( EventBus.INSTANCE.notify(
@ -235,7 +258,7 @@ public class ArbitraryDataManager extends Thread {
arbitraryTransactionData.getIdentifier(), arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(), arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(), arbitraryTransactionData.getService().name(),
"don't need to proactively store, skipping", arbitraryDataExamination.getNotes(),
arbitraryTransactionData.getTimestamp(), arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp() arbitraryTransactionData.getTimestamp()
) )
@ -342,7 +365,7 @@ public class ArbitraryDataManager extends Thread {
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
allArbitraryTransactionsInDescendingOrder allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository() = repository.getArbitraryRepository()
.getLatestArbitraryTransactions(Settings.getInstance().getDataFetchLimit()); .getLatestArbitraryTransactions();
} catch( Exception e) { } catch( Exception e) {
LOGGER.error(e.getMessage(), e); LOGGER.error(e.getMessage(), e);
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0); allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
@ -410,18 +433,6 @@ public class ArbitraryDataManager extends Thread {
// fetch the metadata and notify the event bus again // fetch the metadata and notify the event bus again
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"fetching metadata",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
// Ask our connected peers if they have metadata for this signature // Ask our connected peers if they have metadata for this signature
fetchMetadata(arbitraryTransactionData); fetchMetadata(arbitraryTransactionData);
@ -444,10 +455,14 @@ public class ArbitraryDataManager extends Thread {
} }
} }
private static List<byte[]> processTransactionsForSignatures(int limit, int offset, List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder, Set<ArbitraryTransactionDataHashWrapper> processedTransactions) { private static List<byte[]> processTransactionsForSignatures(
int limit,
int offset,
List<ArbitraryTransactionData> transactionsInDescendingOrder,
Set<ArbitraryTransactionDataHashWrapper> processedTransactions) {
// these transactions are in descending order, latest transactions come first // these transactions are in descending order, latest transactions come first
List<ArbitraryTransactionData> transactions List<ArbitraryTransactionData> transactions
= allArbitraryTransactionsInDescendingOrder.stream() = transactionsInDescendingOrder.stream()
.skip(offset) .skip(offset)
.limit(limit) .limit(limit)
.collect(Collectors.toList()); .collect(Collectors.toList());

24
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
View File

@ -155,31 +155,31 @@ public class ArbitraryDataStorageManager extends Thread {
* @param arbitraryTransactionData - the transaction * @param arbitraryTransactionData - the transaction
* @return boolean - whether to prefetch or not * @return boolean - whether to prefetch or not
*/ */
public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { public ArbitraryDataExamination shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
String name = arbitraryTransactionData.getName(); String name = arbitraryTransactionData.getName();
// Only fetch data associated with hashes, as we already have RAW_DATA // Only fetch data associated with hashes, as we already have RAW_DATA
if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) { if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) {
return false; return new ArbitraryDataExamination(false, "Only fetch data associated with hashes");
} }
// Don't fetch anything more if we're (nearly) out of space // Don't fetch anything more if we're (nearly) out of space
// Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to // Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
// avoid a fetch/delete loop // avoid a fetch/delete loop
if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) { if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) {
return false; return new ArbitraryDataExamination(false,"Don't fetch anything more if we're (nearly) out of space");
} }
// Don't fetch anything if we're (nearly) out of space for this name // Don't fetch anything if we're (nearly) out of space for this name
// Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to // Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
// avoid a fetch/delete loop // avoid a fetch/delete loop
if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) { if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) {
return false; return new ArbitraryDataExamination(false, "Don't fetch anything if we're (nearly) out of space for this name");
} }
// Don't store data unless it's an allowed type (public/private) // Don't store data unless it's an allowed type (public/private)
if (!this.isDataTypeAllowed(arbitraryTransactionData)) { if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
return false; return new ArbitraryDataExamination(false, "Don't store data unless it's an allowed type (public/private)");
} }
// Handle transactions without names differently // Handle transactions without names differently
@ -189,21 +189,21 @@ public class ArbitraryDataStorageManager extends Thread {
// Never fetch data from blocked names, even if they are followed // Never fetch data from blocked names, even if they are followed
if (ListUtils.isNameBlocked(name)) { if (ListUtils.isNameBlocked(name)) {
return false; return new ArbitraryDataExamination(false, "blocked name");
} }
switch (Settings.getInstance().getStoragePolicy()) { switch (Settings.getInstance().getStoragePolicy()) {
case FOLLOWED: case FOLLOWED:
case FOLLOWED_OR_VIEWED: case FOLLOWED_OR_VIEWED:
return ListUtils.isFollowingName(name); return new ArbitraryDataExamination(ListUtils.isFollowingName(name), Settings.getInstance().getStoragePolicy().name());
case ALL: case ALL:
return true; return new ArbitraryDataExamination(true, Settings.getInstance().getStoragePolicy().name());
case NONE: case NONE:
case VIEWED: case VIEWED:
default: default:
return false; return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name());
} }
} }
@ -214,17 +214,17 @@ public class ArbitraryDataStorageManager extends Thread {
* *
* @return boolean - whether the storage policy allows for unnamed data * @return boolean - whether the storage policy allows for unnamed data
*/ */
private boolean shouldPreFetchDataWithoutName() { private ArbitraryDataExamination shouldPreFetchDataWithoutName() {
switch (Settings.getInstance().getStoragePolicy()) { switch (Settings.getInstance().getStoragePolicy()) {
case ALL: case ALL:
return true; return new ArbitraryDataExamination(true, "Fetching all data");
case NONE: case NONE:
case VIEWED: case VIEWED:
case FOLLOWED: case FOLLOWED:
case FOLLOWED_OR_VIEWED: case FOLLOWED_OR_VIEWED:
default: default:
return false; return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name());
} }
} }

8
src/main/java/org/qortal/controller/arbitrary/ArbitraryTransactionDataHashWrapper.java
View File

@ -7,7 +7,7 @@ import java.util.Objects;
public class ArbitraryTransactionDataHashWrapper { public class ArbitraryTransactionDataHashWrapper {
private final ArbitraryTransactionData data; private ArbitraryTransactionData data;
private int service; private int service;
@ -23,6 +23,12 @@ public class ArbitraryTransactionDataHashWrapper {
this.identifier = data.getIdentifier(); this.identifier = data.getIdentifier();
} }
public ArbitraryTransactionDataHashWrapper(int service, String name, String identifier) {
this.service = service;
this.name = name;
this.identifier = identifier;
}
public ArbitraryTransactionData getData() { public ArbitraryTransactionData getData() {
return data; return data;
} }

4
src/main/java/org/qortal/repository/ArbitraryRepository.java
View File

@ -27,7 +27,9 @@ public interface ArbitraryRepository {
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException; public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
List<ArbitraryTransactionData> getLatestArbitraryTransactions(int limit) throws DataException; List<ArbitraryTransactionData> getLatestArbitraryTransactions() throws DataException;
List<ArbitraryTransactionData> getLatestArbitraryTransactionsByName(String name) throws DataException;
public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException; public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException;

76
src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java
View File

@ -228,18 +228,86 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
} }
@Override @Override
public List<ArbitraryTransactionData> getLatestArbitraryTransactions(int limit) throws DataException { public List<ArbitraryTransactionData> getLatestArbitraryTransactions() throws DataException {
String sql = "SELECT type, reference, signature, creator, created_when, fee, " + String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
"tx_group_id, block_height, approval_status, approval_height, " + "tx_group_id, block_height, approval_status, approval_height, " +
"version, nonce, service, size, is_data_raw, data, metadata_hash, " + "version, nonce, service, size, is_data_raw, data, metadata_hash, " +
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " + "name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " + "JOIN Transactions USING (signature) " +
"WHERE name IS NOT NULL " + "WHERE name IS NOT NULL " +
"ORDER BY created_when DESC " + "ORDER BY created_when DESC";
"LIMIT ?";
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>(); List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql, limit)) { try (ResultSet resultSet = this.repository.checkedExecute(sql)) {
if (resultSet == null)
return new ArrayList<>(0);
do {
byte[] reference = resultSet.getBytes(2);
byte[] signature = resultSet.getBytes(3);
byte[] creatorPublicKey = resultSet.getBytes(4);
long timestamp = resultSet.getLong(5);
Long fee = resultSet.getLong(6);
if (fee == 0 && resultSet.wasNull())
fee = null;
int txGroupId = resultSet.getInt(7);
Integer blockHeight = resultSet.getInt(8);
if (blockHeight == 0 && resultSet.wasNull())
blockHeight = null;
ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
Integer approvalHeight = resultSet.getInt(10);
if (approvalHeight == 0 && resultSet.wasNull())
approvalHeight = null;
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
int version = resultSet.getInt(11);
int nonce = resultSet.getInt(12);
int serviceInt = resultSet.getInt(13);
int size = resultSet.getInt(14);
boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
byte[] data = resultSet.getBytes(16);
byte[] metadataHash = resultSet.getBytes(17);
String nameResult = resultSet.getString(18);
String identifierResult = resultSet.getString(19);
Method method = Method.valueOf(resultSet.getInt(20));
byte[] secret = resultSet.getBytes(21);
Compression compression = Compression.valueOf(resultSet.getInt(22));
// FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls.
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
version, serviceInt, nonce, size, nameResult, identifierResult, method, secret,
compression, data, dataType, metadataHash, null);
arbitraryTransactionData.add(transactionData);
} while (resultSet.next());
return arbitraryTransactionData;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
return new ArrayList<>(0);
}
}
@Override
public List<ArbitraryTransactionData> getLatestArbitraryTransactionsByName( String name ) throws DataException {
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
"tx_group_id, block_height, approval_status, approval_height, " +
"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " +
"WHERE name = ? " +
"ORDER BY created_when DESC";
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql, name)) {
if (resultSet == null) if (resultSet == null)
return new ArrayList<>(0); return new ArrayList<>(0);

8
src/main/java/org/qortal/settings/Settings.java
View File

@ -508,9 +508,9 @@ public class Settings {
*/ */
private boolean connectionPoolMonitorEnabled = false; private boolean connectionPoolMonitorEnabled = false;
private int dataFetchLimit = 1_000_000; private int buildArbitraryResourcesBatchSize = 200;
// Domain mapping // Domain mapping
public static class ThreadLimit { public static class ThreadLimit {
private String messageType; private String messageType;
private Integer limit; private Integer limit;
@ -1336,7 +1336,7 @@ public class Settings {
return connectionPoolMonitorEnabled; return connectionPoolMonitorEnabled;
} }
public int getDataFetchLimit() { public int getBuildArbitraryResourcesBatchSize() {
return dataFetchLimit; return buildArbitraryResourcesBatchSize;
} }
} }

142
src/main/java/org/qortal/transaction/ArbitraryTransaction.java
View File

@ -9,6 +9,7 @@ import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Service; import org.qortal.arbitrary.misc.Service;
import org.qortal.block.BlockChain; import org.qortal.block.BlockChain;
import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.controller.arbitrary.ArbitraryTransactionDataHashWrapper;
import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
import org.qortal.crypto.Crypto; import org.qortal.crypto.Crypto;
import org.qortal.crypto.MemoryPoW; import org.qortal.crypto.MemoryPoW;
@ -31,8 +32,12 @@ import org.qortal.utils.ArbitraryTransactionUtils;
import org.qortal.utils.NTP; import org.qortal.utils.NTP;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
public class ArbitraryTransaction extends Transaction { public class ArbitraryTransaction extends Transaction {
@ -303,8 +308,7 @@ public class ArbitraryTransaction extends Transaction {
// Add/update arbitrary resource caches, but don't update the status as this involves time-consuming // Add/update arbitrary resource caches, but don't update the status as this involves time-consuming
// disk reads, and is more prone to failure. The status will be updated on metadata retrieval, or when // disk reads, and is more prone to failure. The status will be updated on metadata retrieval, or when
// accessing the resource. // accessing the resource.
this.updateArbitraryResourceCache(repository); this.updateArbitraryResourceCacheIncludingMetadata(repository, new HashSet<>(0), new HashMap<>(0));
this.updateArbitraryMetadataCache(repository);
repository.saveChanges(); repository.saveChanges();
@ -360,7 +364,10 @@ public class ArbitraryTransaction extends Transaction {
* *
* @throws DataException * @throws DataException
*/ */
public void updateArbitraryResourceCache(Repository repository) throws DataException { public void updateArbitraryResourceCacheIncludingMetadata(
Repository repository,
Set<ArbitraryTransactionDataHashWrapper> latestTransactionWrappers,
Map<ArbitraryTransactionDataHashWrapper, ArbitraryResourceData> resourceByWrapper) throws DataException {
// Don't cache resources without a name (such as auto updates) // Don't cache resources without a name (such as auto updates)
if (arbitraryTransactionData.getName() == null) { if (arbitraryTransactionData.getName() == null) {
return; return;
@ -385,29 +392,42 @@ public class ArbitraryTransaction extends Transaction {
arbitraryResourceData.name = name; arbitraryResourceData.name = name;
arbitraryResourceData.identifier = identifier; arbitraryResourceData.identifier = identifier;
// Get the latest transaction final ArbitraryTransactionDataHashWrapper wrapper = new ArbitraryTransactionDataHashWrapper(arbitraryTransactionData);
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) { ArbitraryTransactionData latestTransactionData;
LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData); if( latestTransactionWrappers.contains(wrapper)) {
// We don't have a latest transaction, so delete from cache latestTransactionData
repository.getArbitraryRepository().delete(arbitraryResourceData); = latestTransactionWrappers.stream()
return; .filter( latestWrapper -> latestWrapper.equals(wrapper))
.findAny().get()
.getData();
} }
else {
// Get the latest transaction
latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) {
LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData);
// We don't have a latest transaction, so delete from cache
repository.getArbitraryRepository().delete(arbitraryResourceData);
return;
}
}
ArbitraryResourceData existingArbitraryResourceData = resourceByWrapper.get(wrapper);
// Get existing cached entry if it exists if( existingArbitraryResourceData == null ) {
ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository() // Get existing cached entry if it exists
.getArbitraryResource(service, name, identifier); existingArbitraryResourceData = repository.getArbitraryRepository()
.getArbitraryResource(service, name, identifier);
LOGGER.info("updating existing arbitraryResourceData" + existingArbitraryResourceData); }
// Check for existing cached data // Check for existing cached data
if (existingArbitraryResourceData == null) { if (existingArbitraryResourceData == null) {
// Nothing exists yet, so set creation date from the current transaction (it will be reduced later if needed) // Nothing exists yet, so set creation date from the current transaction (it will be reduced later if needed)
arbitraryResourceData.created = arbitraryTransactionData.getTimestamp(); arbitraryResourceData.created = arbitraryTransactionData.getTimestamp();
arbitraryResourceData.updated = null; arbitraryResourceData.updated = null;
LOGGER.info("updated = null, reason = existingArbitraryResourceData == null" );
} }
else { else {
resourceByWrapper.put(wrapper, existingArbitraryResourceData);
// An entry already exists - update created time from current transaction if this is older // An entry already exists - update created time from current transaction if this is older
arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp()); arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp());
@ -415,22 +435,44 @@ public class ArbitraryTransaction extends Transaction {
if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) { if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) {
// Latest transaction matches created time, so it hasn't been updated // Latest transaction matches created time, so it hasn't been updated
arbitraryResourceData.updated = null; arbitraryResourceData.updated = null;
LOGGER.info(
"updated = null, reason: existingArbitraryResourceData.created == latestTransactionData.getTimestamp() == " +
existingArbitraryResourceData.created );
} }
else { else {
arbitraryResourceData.updated = latestTransactionData.getTimestamp(); arbitraryResourceData.updated = latestTransactionData.getTimestamp();
LOGGER.info("setting updated to a non-null value");
} }
} }
arbitraryResourceData.size = latestTransactionData.getSize(); arbitraryResourceData.size = latestTransactionData.getSize();
LOGGER.info("saving updated arbitraryResourceData: updated = " + arbitraryResourceData.updated);
// Save // Save
repository.getArbitraryRepository().save(arbitraryResourceData); repository.getArbitraryRepository().save(arbitraryResourceData);
// Update metadata for latest transaction if it is local
if (latestTransactionData.getMetadataHash() != null) {
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
if (metadataFile.exists()) {
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
try {
transactionMetadata.read();
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
metadata.setTitle(transactionMetadata.getTitle());
metadata.setDescription(transactionMetadata.getDescription());
metadata.setCategory(transactionMetadata.getCategory());
metadata.setTags(transactionMetadata.getTags());
repository.getArbitraryRepository().save(metadata);
} catch (IOException e) {
// Ignore, as we can add it again later
}
} else {
// We don't have a local copy of this metadata file, so delete it from the cache
// It will be re-added if the file later arrives via the network
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
repository.getArbitraryRepository().delete(metadata);
}
}
} }
public void updateArbitraryResourceStatus(Repository repository) throws DataException { public void updateArbitraryResourceStatus(Repository repository) throws DataException {
@ -465,60 +507,4 @@ public class ArbitraryTransaction extends Transaction {
repository.getArbitraryRepository().setStatus(arbitraryResourceData, status); repository.getArbitraryRepository().setStatus(arbitraryResourceData, status);
} }
public void updateArbitraryMetadataCache(Repository repository) throws DataException {
// Get the latest transaction
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) {
// We don't have a latest transaction, so give up
return;
}
Service service = latestTransactionData.getService();
String name = latestTransactionData.getName();
String identifier = latestTransactionData.getIdentifier();
if (service == null) {
// Unsupported service - ignore this resource
return;
}
// In the cache we store null identifiers as "default", as it is part of the primary key
if (identifier == null) {
identifier = "default";
}
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.service = service;
arbitraryResourceData.name = name;
arbitraryResourceData.identifier = identifier;
// Update metadata for latest transaction if it is local
if (latestTransactionData.getMetadataHash() != null) {
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
if (metadataFile.exists()) {
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
try {
transactionMetadata.read();
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
metadata.setTitle(transactionMetadata.getTitle());
metadata.setDescription(transactionMetadata.getDescription());
metadata.setCategory(transactionMetadata.getCategory());
metadata.setTags(transactionMetadata.getTags());
repository.getArbitraryRepository().save(metadata);
} catch (IOException e) {
// Ignore, as we can add it again later
}
} else {
// We don't have a local copy of this metadata file, so delete it from the cache
// It will be re-added if the file later arrives via the network
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
repository.getArbitraryRepository().delete(metadata);
}
}
}
} }

15
src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java
View File

@ -209,7 +209,15 @@ public class ArbitraryTransactionUtils {
return ArbitraryTransactionUtils.isFileRecent(filePath, now, cleanupAfter); return ArbitraryTransactionUtils.isFileRecent(filePath, now, cleanupAfter);
} }
public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException { /**
*
* @param arbitraryTransactionData
* @param now
* @param cleanupAfter
* @return true if file is deleted, otherwise return false
* @throws DataException
*/
public static boolean deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
byte[] completeHash = arbitraryTransactionData.getData(); byte[] completeHash = arbitraryTransactionData.getData();
byte[] signature = arbitraryTransactionData.getSignature(); byte[] signature = arbitraryTransactionData.getSignature();
@ -220,6 +228,11 @@ public class ArbitraryTransactionUtils {
"if needed", Base58.encode(completeHash)); "if needed", Base58.encode(completeHash));
arbitraryDataFile.delete(); arbitraryDataFile.delete();
return true;
}
else {
return false;
} }
} }

22
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java
View File

@ -73,14 +73,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store and pre-fetch data for this transaction // We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.FOLLOWED_OR_VIEWED, Settings.getInstance().getStoragePolicy()); assertEquals(StoragePolicy.FOLLOWED_OR_VIEWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(arbitraryTransactionData)); assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name // Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We should store but not pre-fetch data for this transaction // We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(arbitraryTransactionData)); assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
} }
} }
@ -108,14 +108,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store and pre-fetch data for this transaction // We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy()); assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(arbitraryTransactionData)); assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name // Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We shouldn't store or pre-fetch data for this transaction // We shouldn't store or pre-fetch data for this transaction
assertFalse(storageManager.canStoreData(arbitraryTransactionData)); assertFalse(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
} }
} }
@ -143,14 +143,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store but not pre-fetch data for this transaction // We should store but not pre-fetch data for this transaction
assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy()); assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(arbitraryTransactionData)); assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name // Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We should store but not pre-fetch data for this transaction // We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(arbitraryTransactionData)); assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
} }
} }
@ -178,14 +178,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store and pre-fetch data for this transaction // We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy()); assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(arbitraryTransactionData)); assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name // Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We should store and pre-fetch data for this transaction // We should store and pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(arbitraryTransactionData)); assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
} }
} }
@ -213,14 +213,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We shouldn't store or pre-fetch data for this transaction // We shouldn't store or pre-fetch data for this transaction
assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy()); assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy());
assertFalse(storageManager.canStoreData(arbitraryTransactionData)); assertFalse(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name // Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We shouldn't store or pre-fetch data for this transaction // We shouldn't store or pre-fetch data for this transaction
assertFalse(storageManager.canStoreData(arbitraryTransactionData)); assertFalse(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
} }
} }
@ -236,7 +236,7 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store but not pre-fetch data for this transaction // We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(transactionData)); assertTrue(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); assertFalse(storageManager.shouldPreFetchData(repository, transactionData).isPass());
} }
} }