mirror of
https://github.com/Qortal/qortal.git
synced 2025-04-14 23:35:54 +00:00
more arbitrary data optimizations, including the arbitrary resources cache rebuild and a setting to support it, added and removed notifications, added method to the arbitrary repository, also removed an unnecessary setting that was added in the last commit
This commit is contained in:
kennycud
parent
676885ea2d
commit
4baafd1305
@ -2,7 +2,6 @@ package org.qortal.controller.arbitrary;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.qortal.api.resource.TransactionsResource;
|
||||
import org.qortal.controller.Controller;
|
||||
import org.qortal.data.arbitrary.ArbitraryResourceData;
|
||||
import org.qortal.data.transaction.ArbitraryTransactionData;
|
||||
@ -14,12 +13,16 @@ import org.qortal.repository.Repository;
|
||||
import org.qortal.repository.RepositoryManager;
|
||||
import org.qortal.settings.Settings;
|
||||
import org.qortal.transaction.ArbitraryTransaction;
|
||||
import org.qortal.transaction.Transaction;
|
||||
import org.qortal.utils.Base58;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class ArbitraryDataCacheManager extends Thread {
|
||||
|
||||
@ -86,47 +89,10 @@ public class ArbitraryDataCacheManager extends Thread {
|
||||
|
||||
// Update arbitrary resource caches
|
||||
try {
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updating resource cache, queue",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
|
||||
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
|
||||
arbitraryTransaction.updateArbitraryResourceCache(repository);
|
||||
arbitraryTransaction.updateArbitraryMetadataCache(repository);
|
||||
arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, new HashSet<>(0), new HashMap<>(0));
|
||||
repository.saveChanges();
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updated resource cache",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updating resource status, queue",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
|
||||
// Update status as separate commit, as this is more prone to failure
|
||||
arbitraryTransaction.updateArbitraryResourceStatus(repository);
|
||||
repository.saveChanges();
|
||||
@ -137,7 +103,7 @@ public class ArbitraryDataCacheManager extends Thread {
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updated resource status",
|
||||
"updated resource cache and status, queue",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
@ -201,64 +167,61 @@ public class ArbitraryDataCacheManager extends Thread {
|
||||
LOGGER.info("Building arbitrary resources cache...");
|
||||
SplashFrame.getInstance().updateStatus("Building QDN cache - please wait...");
|
||||
|
||||
final int batchSize = 100;
|
||||
final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
|
||||
int offset = 0;
|
||||
|
||||
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder
|
||||
= repository.getArbitraryRepository().getLatestArbitraryTransactions();
|
||||
|
||||
LOGGER.info("arbitrary transactions: count = " + allArbitraryTransactionsInDescendingOrder.size());
|
||||
|
||||
List<ArbitraryResourceData> resources = repository.getArbitraryRepository().getArbitraryResources(null, null, true);
|
||||
|
||||
Map<ArbitraryTransactionDataHashWrapper, ArbitraryResourceData> resourceByWrapper = new HashMap<>(resources.size());
|
||||
for( ArbitraryResourceData resource : resources ) {
|
||||
resourceByWrapper.put(
|
||||
new ArbitraryTransactionDataHashWrapper(resource.service.value, resource.name, resource.identifier),
|
||||
resource
|
||||
);
|
||||
}
|
||||
|
||||
LOGGER.info("arbitrary resources: count = " + resourceByWrapper.size());
|
||||
|
||||
Set<ArbitraryTransactionDataHashWrapper> latestTransactionsWrapped = new HashSet<>(allArbitraryTransactionsInDescendingOrder.size());
|
||||
|
||||
// Loop through all ARBITRARY transactions, and determine latest state
|
||||
while (!Controller.isStopping()) {
|
||||
LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1);
|
||||
|
||||
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false);
|
||||
if (signatures.isEmpty()) {
|
||||
List<ArbitraryTransactionData> transactionsToProcess
|
||||
= allArbitraryTransactionsInDescendingOrder.stream()
|
||||
.skip(offset)
|
||||
.limit(batchSize)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if (transactionsToProcess.isEmpty()) {
|
||||
// Complete
|
||||
break;
|
||||
}
|
||||
|
||||
// Expand signatures to transactions
|
||||
for (byte[] signature : signatures) {
|
||||
try {
|
||||
ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository
|
||||
.getTransactionRepository().fromSignature(signature);
|
||||
|
||||
try {
|
||||
for( ArbitraryTransactionData transactionData : transactionsToProcess) {
|
||||
if (transactionData.getService() == null) {
|
||||
// Unsupported service - ignore this resource
|
||||
continue;
|
||||
}
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updating resource cache, build",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
latestTransactionsWrapped.add(new ArbitraryTransactionDataHashWrapper(transactionData));
|
||||
|
||||
// Update arbitrary resource caches
|
||||
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
|
||||
arbitraryTransaction.updateArbitraryResourceCache(repository);
|
||||
arbitraryTransaction.updateArbitraryMetadataCache(repository);
|
||||
repository.saveChanges();
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updated resource cache",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
} catch (DataException e) {
|
||||
repository.discardChanges();
|
||||
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, latestTransactionsWrapped, resourceByWrapper);
|
||||
}
|
||||
repository.saveChanges();
|
||||
} catch (DataException e) {
|
||||
repository.discardChanges();
|
||||
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
}
|
||||
offset += batchSize;
|
||||
}
|
||||
@ -288,7 +251,7 @@ public class ArbitraryDataCacheManager extends Thread {
|
||||
LOGGER.info("Refreshing arbitrary resource statuses for locally hosted transactions...");
|
||||
SplashFrame.getInstance().updateStatus("Refreshing statuses - please wait...");
|
||||
|
||||
final int batchSize = 100;
|
||||
final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
|
||||
int offset = 0;
|
||||
|
||||
// Loop through all ARBITRARY transactions, and determine latest state
|
||||
@ -301,45 +264,21 @@ public class ArbitraryDataCacheManager extends Thread {
|
||||
break;
|
||||
}
|
||||
|
||||
// Loop through hosted transactions
|
||||
for (ArbitraryTransactionData transactionData : hostedTransactions) {
|
||||
|
||||
try {
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updating resource status",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
try {
|
||||
// Loop through hosted transactions
|
||||
for (ArbitraryTransactionData transactionData : hostedTransactions) {
|
||||
|
||||
// Determine status and update cache
|
||||
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
|
||||
arbitraryTransaction.updateArbitraryResourceStatus(repository);
|
||||
repository.saveChanges();
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updated resource status",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
|
||||
} catch (DataException e) {
|
||||
repository.discardChanges();
|
||||
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
}
|
||||
repository.saveChanges();
|
||||
} catch (DataException e) {
|
||||
repository.discardChanges();
|
||||
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
offset += batchSize;
|
||||
}
|
||||
|
||||
|
||||
@ -2,7 +2,6 @@ package org.qortal.controller.arbitrary;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
|
||||
import org.qortal.data.transaction.ArbitraryTransactionData;
|
||||
import org.qortal.data.transaction.TransactionData;
|
||||
import org.qortal.event.DataMonitorEvent;
|
||||
@ -23,9 +22,12 @@ import java.nio.file.Paths;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD;
|
||||
|
||||
@ -80,6 +82,19 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
final int limit = 100;
|
||||
int offset = 0;
|
||||
|
||||
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
|
||||
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
allArbitraryTransactionsInDescendingOrder
|
||||
= repository.getArbitraryRepository()
|
||||
.getLatestArbitraryTransactions();
|
||||
} catch( Exception e) {
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
|
||||
}
|
||||
|
||||
Set<ArbitraryTransactionData> processedTransactions = new HashSet<>();
|
||||
|
||||
try {
|
||||
while (!isStopping) {
|
||||
Thread.sleep(30000);
|
||||
@ -110,27 +125,31 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
|
||||
// Any arbitrary transactions we want to fetch data for?
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true);
|
||||
// LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
|
||||
List<ArbitraryTransactionData> transactions = allArbitraryTransactionsInDescendingOrder.stream().skip(offset).limit(limit).collect(Collectors.toList());
|
||||
if (isStopping) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (signatures == null || signatures.isEmpty()) {
|
||||
if (transactions == null || transactions.isEmpty()) {
|
||||
offset = 0;
|
||||
continue;
|
||||
allArbitraryTransactionsInDescendingOrder
|
||||
= repository.getArbitraryRepository()
|
||||
.getLatestArbitraryTransactions();
|
||||
transactions = allArbitraryTransactionsInDescendingOrder.stream().limit(limit).collect(Collectors.toList());
|
||||
processedTransactions.clear();
|
||||
}
|
||||
|
||||
offset += limit;
|
||||
now = NTP.getTime();
|
||||
|
||||
// Loop through the signatures in this batch
|
||||
for (int i=0; i<signatures.size(); i++) {
|
||||
for (int i=0; i<transactions.size(); i++) {
|
||||
if (isStopping) {
|
||||
return;
|
||||
}
|
||||
|
||||
byte[] signature = signatures.get(i);
|
||||
if (signature == null) {
|
||||
ArbitraryTransactionData arbitraryTransactionData = transactions.get(i);
|
||||
if (arbitraryTransactionData == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -139,9 +158,7 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
Thread.sleep(5000);
|
||||
}
|
||||
|
||||
// Fetch the transaction data
|
||||
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
|
||||
if (arbitraryTransactionData == null || arbitraryTransactionData.getService() == null) {
|
||||
if (arbitraryTransactionData.getService() == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -150,6 +167,8 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
continue;
|
||||
}
|
||||
|
||||
boolean mostRecentTransaction = processedTransactions.add(arbitraryTransactionData);
|
||||
|
||||
// Check if we have the complete file
|
||||
boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData);
|
||||
|
||||
@ -186,28 +205,38 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
}
|
||||
|
||||
// Check to see if we have had a more recent PUT
|
||||
Optional<ArbitraryTransactionData> moreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
|
||||
boolean hasMoreRecentPutTransaction = moreRecentPutTransaction.isPresent();
|
||||
if (hasMoreRecentPutTransaction) {
|
||||
if (!mostRecentTransaction) {
|
||||
// There is a more recent PUT transaction than the one we are currently processing.
|
||||
// When a PUT is issued, it replaces any layers that would have been there before.
|
||||
// Therefore any data relating to this older transaction is no longer needed.
|
||||
LOGGER.info(String.format("Newer PUT found for %s %s since transaction %s. " +
|
||||
"Deleting all files associated with the earlier transaction.", arbitraryTransactionData.getService(),
|
||||
arbitraryTransactionData.getName(), Base58.encode(signature)));
|
||||
arbitraryTransactionData.getName(), Base58.encode(arbitraryTransactionData.getSignature())));
|
||||
|
||||
ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"deleting data due to replacement",
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
moreRecentPutTransaction.get().getTimestamp()
|
||||
)
|
||||
);
|
||||
|
||||
Optional<ArbitraryTransactionData> moreRecentPutTransaction
|
||||
= processedTransactions.stream()
|
||||
.filter(data -> data.equals(arbitraryTransactionData))
|
||||
.findAny();
|
||||
|
||||
if( moreRecentPutTransaction.isPresent() ) {
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"deleting data due to replacement",
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
moreRecentPutTransaction.get().getTimestamp()
|
||||
)
|
||||
);
|
||||
}
|
||||
else {
|
||||
LOGGER.warn("Something went wrong with the most recent put transaction determination!");
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -226,18 +255,21 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
LOGGER.debug(String.format("Transaction %s has complete file and all chunks",
|
||||
Base58.encode(arbitraryTransactionData.getSignature())));
|
||||
|
||||
ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"deleting data due to age",
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
arbitraryTransactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
boolean wasDeleted = ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
|
||||
|
||||
if( wasDeleted ) {
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"deleting file, retaining chunks",
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
arbitraryTransactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -452,18 +484,6 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
// Relates to a different name - don't delete it
|
||||
return false;
|
||||
}
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"randomly selected for deletion",
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
arbitraryTransactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
} catch (DataException e) {
|
||||
@ -473,6 +493,7 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
}
|
||||
|
||||
LOGGER.info("Deleting random file {} because we have reached max storage capacity...", randomItem.toString());
|
||||
fireRandomItemDeletionNotification(randomItem, repository, "Deleting random file, because we have reached max storage capacity");
|
||||
boolean success = randomItem.delete();
|
||||
if (success) {
|
||||
try {
|
||||
@ -487,6 +508,35 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
return false;
|
||||
}
|
||||
|
||||
private void fireRandomItemDeletionNotification(File randomItem, Repository repository, String reason) {
|
||||
try {
|
||||
Path parentFileNamePath = randomItem.toPath().toAbsolutePath().getParent().getFileName();
|
||||
if (parentFileNamePath != null) {
|
||||
String signature58 = parentFileNamePath.toString();
|
||||
byte[] signature = Base58.decode(signature58);
|
||||
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
|
||||
if (transactionData != null && transactionData.getType() == Transaction.TransactionType.ARBITRARY) {
|
||||
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
reason,
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
arbitraryTransactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanupTempDirectory(String folder, long now, long minAge) {
|
||||
String baseDir = Settings.getInstance().getTempDataPath();
|
||||
Path tempDir = Paths.get(baseDir, folder);
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
package org.qortal.controller.arbitrary;
|
||||
|
||||
public class ArbitraryDataExamination {
|
||||
|
||||
private boolean pass;
|
||||
|
||||
private String notes;
|
||||
|
||||
public ArbitraryDataExamination(boolean pass, String notes) {
|
||||
this.pass = pass;
|
||||
this.notes = notes;
|
||||
}
|
||||
|
||||
public boolean isPass() {
|
||||
return pass;
|
||||
}
|
||||
|
||||
public String getNotes() {
|
||||
return notes;
|
||||
}
|
||||
}
|
||||
@ -198,13 +198,35 @@ public class ArbitraryDataManager extends Thread {
|
||||
final int limit = 100;
|
||||
int offset = 0;
|
||||
|
||||
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
|
||||
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
|
||||
if( name == null ) {
|
||||
allArbitraryTransactionsInDescendingOrder
|
||||
= repository.getArbitraryRepository()
|
||||
.getLatestArbitraryTransactions();
|
||||
}
|
||||
else {
|
||||
allArbitraryTransactionsInDescendingOrder
|
||||
= repository.getArbitraryRepository()
|
||||
.getLatestArbitraryTransactionsByName(name);
|
||||
}
|
||||
} catch( Exception e) {
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
|
||||
}
|
||||
|
||||
// collect processed transactions in a set to ensure outdated data transactions do not get fetched
|
||||
Set<ArbitraryTransactionDataHashWrapper> processedTransactions = new HashSet<>();
|
||||
|
||||
while (!isStopping) {
|
||||
Thread.sleep(1000L);
|
||||
|
||||
// Any arbitrary transactions we want to fetch data for?
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, name, null, ConfirmationStatus.BOTH, limit, offset, true);
|
||||
// LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
|
||||
List<byte[]> signatures = processTransactionsForSignatures(limit, offset, allArbitraryTransactionsInDescendingOrder, processedTransactions);
|
||||
|
||||
if (signatures == null || signatures.isEmpty()) {
|
||||
offset = 0;
|
||||
break;
|
||||
@ -226,7 +248,8 @@ public class ArbitraryDataManager extends Thread {
|
||||
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
|
||||
|
||||
// Skip transactions that we don't need to proactively store data for
|
||||
if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) {
|
||||
ArbitraryDataExamination arbitraryDataExamination = storageManager.shouldPreFetchData(repository, arbitraryTransactionData);
|
||||
if (!arbitraryDataExamination.isPass()) {
|
||||
iterator.remove();
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
@ -235,7 +258,7 @@ public class ArbitraryDataManager extends Thread {
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"don't need to proactively store, skipping",
|
||||
arbitraryDataExamination.getNotes(),
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
arbitraryTransactionData.getTimestamp()
|
||||
)
|
||||
@ -342,7 +365,7 @@ public class ArbitraryDataManager extends Thread {
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
allArbitraryTransactionsInDescendingOrder
|
||||
= repository.getArbitraryRepository()
|
||||
.getLatestArbitraryTransactions(Settings.getInstance().getDataFetchLimit());
|
||||
.getLatestArbitraryTransactions();
|
||||
} catch( Exception e) {
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
|
||||
@ -410,18 +433,6 @@ public class ArbitraryDataManager extends Thread {
|
||||
// fetch the metadata and notify the event bus again
|
||||
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"fetching metadata",
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
arbitraryTransactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
|
||||
// Ask our connected peers if they have metadata for this signature
|
||||
fetchMetadata(arbitraryTransactionData);
|
||||
|
||||
@ -444,10 +455,14 @@ public class ArbitraryDataManager extends Thread {
|
||||
}
|
||||
}
|
||||
|
||||
private static List<byte[]> processTransactionsForSignatures(int limit, int offset, List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder, Set<ArbitraryTransactionDataHashWrapper> processedTransactions) {
|
||||
private static List<byte[]> processTransactionsForSignatures(
|
||||
int limit,
|
||||
int offset,
|
||||
List<ArbitraryTransactionData> transactionsInDescendingOrder,
|
||||
Set<ArbitraryTransactionDataHashWrapper> processedTransactions) {
|
||||
// these transactions are in descending order, latest transactions come first
|
||||
List<ArbitraryTransactionData> transactions
|
||||
= allArbitraryTransactionsInDescendingOrder.stream()
|
||||
= transactionsInDescendingOrder.stream()
|
||||
.skip(offset)
|
||||
.limit(limit)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
@ -155,31 +155,31 @@ public class ArbitraryDataStorageManager extends Thread {
|
||||
* @param arbitraryTransactionData - the transaction
|
||||
* @return boolean - whether to prefetch or not
|
||||
*/
|
||||
public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
|
||||
public ArbitraryDataExamination shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
|
||||
String name = arbitraryTransactionData.getName();
|
||||
|
||||
// Only fetch data associated with hashes, as we already have RAW_DATA
|
||||
if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) {
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false, "Only fetch data associated with hashes");
|
||||
}
|
||||
|
||||
// Don't fetch anything more if we're (nearly) out of space
|
||||
// Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
|
||||
// avoid a fetch/delete loop
|
||||
if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) {
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false,"Don't fetch anything more if we're (nearly) out of space");
|
||||
}
|
||||
|
||||
// Don't fetch anything if we're (nearly) out of space for this name
|
||||
// Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
|
||||
// avoid a fetch/delete loop
|
||||
if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) {
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false, "Don't fetch anything if we're (nearly) out of space for this name");
|
||||
}
|
||||
|
||||
// Don't store data unless it's an allowed type (public/private)
|
||||
if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false, "Don't store data unless it's an allowed type (public/private)");
|
||||
}
|
||||
|
||||
// Handle transactions without names differently
|
||||
@ -189,21 +189,21 @@ public class ArbitraryDataStorageManager extends Thread {
|
||||
|
||||
// Never fetch data from blocked names, even if they are followed
|
||||
if (ListUtils.isNameBlocked(name)) {
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false, "blocked name");
|
||||
}
|
||||
|
||||
switch (Settings.getInstance().getStoragePolicy()) {
|
||||
case FOLLOWED:
|
||||
case FOLLOWED_OR_VIEWED:
|
||||
return ListUtils.isFollowingName(name);
|
||||
return new ArbitraryDataExamination(ListUtils.isFollowingName(name), Settings.getInstance().getStoragePolicy().name());
|
||||
|
||||
case ALL:
|
||||
return true;
|
||||
return new ArbitraryDataExamination(true, Settings.getInstance().getStoragePolicy().name());
|
||||
|
||||
case NONE:
|
||||
case VIEWED:
|
||||
default:
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name());
|
||||
}
|
||||
}
|
||||
|
||||
@ -214,17 +214,17 @@ public class ArbitraryDataStorageManager extends Thread {
|
||||
*
|
||||
* @return boolean - whether the storage policy allows for unnamed data
|
||||
*/
|
||||
private boolean shouldPreFetchDataWithoutName() {
|
||||
private ArbitraryDataExamination shouldPreFetchDataWithoutName() {
|
||||
switch (Settings.getInstance().getStoragePolicy()) {
|
||||
case ALL:
|
||||
return true;
|
||||
return new ArbitraryDataExamination(true, "Fetching all data");
|
||||
|
||||
case NONE:
|
||||
case VIEWED:
|
||||
case FOLLOWED:
|
||||
case FOLLOWED_OR_VIEWED:
|
||||
default:
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ import java.util.Objects;
|
||||
|
||||
public class ArbitraryTransactionDataHashWrapper {
|
||||
|
||||
private final ArbitraryTransactionData data;
|
||||
private ArbitraryTransactionData data;
|
||||
|
||||
private int service;
|
||||
|
||||
@ -23,6 +23,12 @@ public class ArbitraryTransactionDataHashWrapper {
|
||||
this.identifier = data.getIdentifier();
|
||||
}
|
||||
|
||||
public ArbitraryTransactionDataHashWrapper(int service, String name, String identifier) {
|
||||
this.service = service;
|
||||
this.name = name;
|
||||
this.identifier = identifier;
|
||||
}
|
||||
|
||||
public ArbitraryTransactionData getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
@ -27,7 +27,9 @@ public interface ArbitraryRepository {
|
||||
|
||||
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
|
||||
|
||||
List<ArbitraryTransactionData> getLatestArbitraryTransactions(int limit) throws DataException;
|
||||
List<ArbitraryTransactionData> getLatestArbitraryTransactions() throws DataException;
|
||||
|
||||
List<ArbitraryTransactionData> getLatestArbitraryTransactionsByName(String name) throws DataException;
|
||||
|
||||
public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException;
|
||||
|
||||
|
||||
@ -227,18 +227,86 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ArbitraryTransactionData> getLatestArbitraryTransactions(int limit) throws DataException {
|
||||
public List<ArbitraryTransactionData> getLatestArbitraryTransactions() throws DataException {
|
||||
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
|
||||
"tx_group_id, block_height, approval_status, approval_height, " +
|
||||
"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
|
||||
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
|
||||
"JOIN Transactions USING (signature) " +
|
||||
"WHERE name IS NOT NULL " +
|
||||
"ORDER BY created_when DESC " +
|
||||
"LIMIT ?";
|
||||
"ORDER BY created_when DESC";
|
||||
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
|
||||
|
||||
try (ResultSet resultSet = this.repository.checkedExecute(sql, limit)) {
|
||||
try (ResultSet resultSet = this.repository.checkedExecute(sql)) {
|
||||
if (resultSet == null)
|
||||
return new ArrayList<>(0);
|
||||
|
||||
do {
|
||||
byte[] reference = resultSet.getBytes(2);
|
||||
byte[] signature = resultSet.getBytes(3);
|
||||
byte[] creatorPublicKey = resultSet.getBytes(4);
|
||||
long timestamp = resultSet.getLong(5);
|
||||
|
||||
Long fee = resultSet.getLong(6);
|
||||
if (fee == 0 && resultSet.wasNull())
|
||||
fee = null;
|
||||
|
||||
int txGroupId = resultSet.getInt(7);
|
||||
|
||||
Integer blockHeight = resultSet.getInt(8);
|
||||
if (blockHeight == 0 && resultSet.wasNull())
|
||||
blockHeight = null;
|
||||
|
||||
ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
|
||||
Integer approvalHeight = resultSet.getInt(10);
|
||||
if (approvalHeight == 0 && resultSet.wasNull())
|
||||
approvalHeight = null;
|
||||
|
||||
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
|
||||
|
||||
int version = resultSet.getInt(11);
|
||||
int nonce = resultSet.getInt(12);
|
||||
int serviceInt = resultSet.getInt(13);
|
||||
int size = resultSet.getInt(14);
|
||||
boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
|
||||
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
|
||||
byte[] data = resultSet.getBytes(16);
|
||||
byte[] metadataHash = resultSet.getBytes(17);
|
||||
String nameResult = resultSet.getString(18);
|
||||
String identifierResult = resultSet.getString(19);
|
||||
Method method = Method.valueOf(resultSet.getInt(20));
|
||||
byte[] secret = resultSet.getBytes(21);
|
||||
Compression compression = Compression.valueOf(resultSet.getInt(22));
|
||||
// FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls.
|
||||
|
||||
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
|
||||
version, serviceInt, nonce, size, nameResult, identifierResult, method, secret,
|
||||
compression, data, dataType, metadataHash, null);
|
||||
|
||||
arbitraryTransactionData.add(transactionData);
|
||||
} while (resultSet.next());
|
||||
|
||||
return arbitraryTransactionData;
|
||||
} catch (SQLException e) {
|
||||
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
return new ArrayList<>(0);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ArbitraryTransactionData> getLatestArbitraryTransactionsByName( String name ) throws DataException {
|
||||
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
|
||||
"tx_group_id, block_height, approval_status, approval_height, " +
|
||||
"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
|
||||
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
|
||||
"JOIN Transactions USING (signature) " +
|
||||
"WHERE name = ? " +
|
||||
"ORDER BY created_when DESC";
|
||||
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
|
||||
|
||||
try (ResultSet resultSet = this.repository.checkedExecute(sql, name)) {
|
||||
if (resultSet == null)
|
||||
return new ArrayList<>(0);
|
||||
|
||||
|
||||
@ -508,9 +508,9 @@ public class Settings {
|
||||
*/
|
||||
private boolean connectionPoolMonitorEnabled = false;
|
||||
|
||||
private int dataFetchLimit = 1_000_000;
|
||||
private int buildArbitraryResourcesBatchSize = 200;
|
||||
|
||||
// Domain mapping
|
||||
// Domain mapping
|
||||
public static class ThreadLimit {
|
||||
private String messageType;
|
||||
private Integer limit;
|
||||
@ -1336,7 +1336,7 @@ public class Settings {
|
||||
return connectionPoolMonitorEnabled;
|
||||
}
|
||||
|
||||
public int getDataFetchLimit() {
|
||||
return dataFetchLimit;
|
||||
public int getBuildArbitraryResourcesBatchSize() {
|
||||
return buildArbitraryResourcesBatchSize;
|
||||
}
|
||||
}
|
||||
|
||||
@ -9,6 +9,7 @@ import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
|
||||
import org.qortal.arbitrary.misc.Service;
|
||||
import org.qortal.block.BlockChain;
|
||||
import org.qortal.controller.arbitrary.ArbitraryDataManager;
|
||||
import org.qortal.controller.arbitrary.ArbitraryTransactionDataHashWrapper;
|
||||
import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
|
||||
import org.qortal.crypto.Crypto;
|
||||
import org.qortal.crypto.MemoryPoW;
|
||||
@ -31,8 +32,12 @@ import org.qortal.utils.ArbitraryTransactionUtils;
|
||||
import org.qortal.utils.NTP;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class ArbitraryTransaction extends Transaction {
|
||||
@ -303,8 +308,7 @@ public class ArbitraryTransaction extends Transaction {
|
||||
// Add/update arbitrary resource caches, but don't update the status as this involves time-consuming
|
||||
// disk reads, and is more prone to failure. The status will be updated on metadata retrieval, or when
|
||||
// accessing the resource.
|
||||
this.updateArbitraryResourceCache(repository);
|
||||
this.updateArbitraryMetadataCache(repository);
|
||||
this.updateArbitraryResourceCacheIncludingMetadata(repository, new HashSet<>(0), new HashMap<>(0));
|
||||
|
||||
repository.saveChanges();
|
||||
|
||||
@ -360,7 +364,10 @@ public class ArbitraryTransaction extends Transaction {
|
||||
*
|
||||
* @throws DataException
|
||||
*/
|
||||
public void updateArbitraryResourceCache(Repository repository) throws DataException {
|
||||
public void updateArbitraryResourceCacheIncludingMetadata(
|
||||
Repository repository,
|
||||
Set<ArbitraryTransactionDataHashWrapper> latestTransactionWrappers,
|
||||
Map<ArbitraryTransactionDataHashWrapper, ArbitraryResourceData> resourceByWrapper) throws DataException {
|
||||
// Don't cache resources without a name (such as auto updates)
|
||||
if (arbitraryTransactionData.getName() == null) {
|
||||
return;
|
||||
@ -385,29 +392,42 @@ public class ArbitraryTransaction extends Transaction {
|
||||
arbitraryResourceData.name = name;
|
||||
arbitraryResourceData.identifier = identifier;
|
||||
|
||||
// Get the latest transaction
|
||||
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
|
||||
if (latestTransactionData == null) {
|
||||
LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData);
|
||||
// We don't have a latest transaction, so delete from cache
|
||||
repository.getArbitraryRepository().delete(arbitraryResourceData);
|
||||
return;
|
||||
final ArbitraryTransactionDataHashWrapper wrapper = new ArbitraryTransactionDataHashWrapper(arbitraryTransactionData);
|
||||
|
||||
ArbitraryTransactionData latestTransactionData;
|
||||
if( latestTransactionWrappers.contains(wrapper)) {
|
||||
latestTransactionData
|
||||
= latestTransactionWrappers.stream()
|
||||
.filter( latestWrapper -> latestWrapper.equals(wrapper))
|
||||
.findAny().get()
|
||||
.getData();
|
||||
}
|
||||
else {
|
||||
// Get the latest transaction
|
||||
latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
|
||||
if (latestTransactionData == null) {
|
||||
LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData);
|
||||
// We don't have a latest transaction, so delete from cache
|
||||
repository.getArbitraryRepository().delete(arbitraryResourceData);
|
||||
return;
|
||||
}
|
||||
}
|
||||
ArbitraryResourceData existingArbitraryResourceData = resourceByWrapper.get(wrapper);
|
||||
|
||||
// Get existing cached entry if it exists
|
||||
ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository()
|
||||
.getArbitraryResource(service, name, identifier);
|
||||
|
||||
LOGGER.info("updating existing arbitraryResourceData" + existingArbitraryResourceData);
|
||||
if( existingArbitraryResourceData == null ) {
|
||||
// Get existing cached entry if it exists
|
||||
existingArbitraryResourceData = repository.getArbitraryRepository()
|
||||
.getArbitraryResource(service, name, identifier);
|
||||
}
|
||||
|
||||
// Check for existing cached data
|
||||
if (existingArbitraryResourceData == null) {
|
||||
// Nothing exists yet, so set creation date from the current transaction (it will be reduced later if needed)
|
||||
arbitraryResourceData.created = arbitraryTransactionData.getTimestamp();
|
||||
arbitraryResourceData.updated = null;
|
||||
LOGGER.info("updated = null, reason = existingArbitraryResourceData == null" );
|
||||
}
|
||||
else {
|
||||
resourceByWrapper.put(wrapper, existingArbitraryResourceData);
|
||||
// An entry already exists - update created time from current transaction if this is older
|
||||
arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp());
|
||||
|
||||
@ -415,22 +435,44 @@ public class ArbitraryTransaction extends Transaction {
|
||||
if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) {
|
||||
// Latest transaction matches created time, so it hasn't been updated
|
||||
arbitraryResourceData.updated = null;
|
||||
LOGGER.info(
|
||||
"updated = null, reason: existingArbitraryResourceData.created == latestTransactionData.getTimestamp() == " +
|
||||
existingArbitraryResourceData.created );
|
||||
}
|
||||
else {
|
||||
arbitraryResourceData.updated = latestTransactionData.getTimestamp();
|
||||
LOGGER.info("setting updated to a non-null value");
|
||||
}
|
||||
}
|
||||
|
||||
arbitraryResourceData.size = latestTransactionData.getSize();
|
||||
|
||||
LOGGER.info("saving updated arbitraryResourceData: updated = " + arbitraryResourceData.updated);
|
||||
|
||||
// Save
|
||||
repository.getArbitraryRepository().save(arbitraryResourceData);
|
||||
|
||||
// Update metadata for latest transaction if it is local
|
||||
if (latestTransactionData.getMetadataHash() != null) {
|
||||
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
|
||||
if (metadataFile.exists()) {
|
||||
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
|
||||
try {
|
||||
transactionMetadata.read();
|
||||
|
||||
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
|
||||
metadata.setArbitraryResourceData(arbitraryResourceData);
|
||||
metadata.setTitle(transactionMetadata.getTitle());
|
||||
metadata.setDescription(transactionMetadata.getDescription());
|
||||
metadata.setCategory(transactionMetadata.getCategory());
|
||||
metadata.setTags(transactionMetadata.getTags());
|
||||
repository.getArbitraryRepository().save(metadata);
|
||||
|
||||
} catch (IOException e) {
|
||||
// Ignore, as we can add it again later
|
||||
}
|
||||
} else {
|
||||
// We don't have a local copy of this metadata file, so delete it from the cache
|
||||
// It will be re-added if the file later arrives via the network
|
||||
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
|
||||
metadata.setArbitraryResourceData(arbitraryResourceData);
|
||||
repository.getArbitraryRepository().delete(metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void updateArbitraryResourceStatus(Repository repository) throws DataException {
|
||||
@ -465,60 +507,4 @@ public class ArbitraryTransaction extends Transaction {
|
||||
repository.getArbitraryRepository().setStatus(arbitraryResourceData, status);
|
||||
}
|
||||
|
||||
public void updateArbitraryMetadataCache(Repository repository) throws DataException {
|
||||
// Get the latest transaction
|
||||
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
|
||||
if (latestTransactionData == null) {
|
||||
// We don't have a latest transaction, so give up
|
||||
return;
|
||||
}
|
||||
|
||||
Service service = latestTransactionData.getService();
|
||||
String name = latestTransactionData.getName();
|
||||
String identifier = latestTransactionData.getIdentifier();
|
||||
|
||||
if (service == null) {
|
||||
// Unsupported service - ignore this resource
|
||||
return;
|
||||
}
|
||||
|
||||
// In the cache we store null identifiers as "default", as it is part of the primary key
|
||||
if (identifier == null) {
|
||||
identifier = "default";
|
||||
}
|
||||
|
||||
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
|
||||
arbitraryResourceData.service = service;
|
||||
arbitraryResourceData.name = name;
|
||||
arbitraryResourceData.identifier = identifier;
|
||||
|
||||
// Update metadata for latest transaction if it is local
|
||||
if (latestTransactionData.getMetadataHash() != null) {
|
||||
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
|
||||
if (metadataFile.exists()) {
|
||||
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
|
||||
try {
|
||||
transactionMetadata.read();
|
||||
|
||||
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
|
||||
metadata.setArbitraryResourceData(arbitraryResourceData);
|
||||
metadata.setTitle(transactionMetadata.getTitle());
|
||||
metadata.setDescription(transactionMetadata.getDescription());
|
||||
metadata.setCategory(transactionMetadata.getCategory());
|
||||
metadata.setTags(transactionMetadata.getTags());
|
||||
repository.getArbitraryRepository().save(metadata);
|
||||
|
||||
} catch (IOException e) {
|
||||
// Ignore, as we can add it again later
|
||||
}
|
||||
} else {
|
||||
// We don't have a local copy of this metadata file, so delete it from the cache
|
||||
// It will be re-added if the file later arrives via the network
|
||||
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
|
||||
metadata.setArbitraryResourceData(arbitraryResourceData);
|
||||
repository.getArbitraryRepository().delete(metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -209,7 +209,15 @@ public class ArbitraryTransactionUtils {
|
||||
return ArbitraryTransactionUtils.isFileRecent(filePath, now, cleanupAfter);
|
||||
}
|
||||
|
||||
public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
|
||||
/**
|
||||
*
|
||||
* @param arbitraryTransactionData
|
||||
* @param now
|
||||
* @param cleanupAfter
|
||||
* @return true if file is deleted, otherwise return false
|
||||
* @throws DataException
|
||||
*/
|
||||
public static boolean deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
|
||||
byte[] completeHash = arbitraryTransactionData.getData();
|
||||
byte[] signature = arbitraryTransactionData.getSignature();
|
||||
|
||||
@ -220,6 +228,11 @@ public class ArbitraryTransactionUtils {
|
||||
"if needed", Base58.encode(completeHash));
|
||||
|
||||
arbitraryDataFile.delete();
|
||||
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -73,14 +73,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
|
||||
// We should store and pre-fetch data for this transaction
|
||||
assertEquals(StoragePolicy.FOLLOWED_OR_VIEWED, Settings.getInstance().getStoragePolicy());
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
|
||||
// Now unfollow the name
|
||||
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
|
||||
|
||||
// We should store but not pre-fetch data for this transaction
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
}
|
||||
}
|
||||
|
||||
@ -108,14 +108,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
|
||||
// We should store and pre-fetch data for this transaction
|
||||
assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy());
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
|
||||
// Now unfollow the name
|
||||
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
|
||||
|
||||
// We shouldn't store or pre-fetch data for this transaction
|
||||
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
}
|
||||
}
|
||||
|
||||
@ -143,14 +143,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
|
||||
// We should store but not pre-fetch data for this transaction
|
||||
assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy());
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
|
||||
// Now unfollow the name
|
||||
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
|
||||
|
||||
// We should store but not pre-fetch data for this transaction
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
}
|
||||
}
|
||||
|
||||
@ -178,14 +178,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
|
||||
// We should store and pre-fetch data for this transaction
|
||||
assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy());
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
|
||||
// Now unfollow the name
|
||||
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
|
||||
|
||||
// We should store and pre-fetch data for this transaction
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
}
|
||||
}
|
||||
|
||||
@ -213,14 +213,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
|
||||
// We shouldn't store or pre-fetch data for this transaction
|
||||
assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy());
|
||||
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
|
||||
// Now unfollow the name
|
||||
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
|
||||
|
||||
// We shouldn't store or pre-fetch data for this transaction
|
||||
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
}
|
||||
}
|
||||
|
||||
@ -236,7 +236,7 @@ public class ArbitraryDataStoragePolicyTests extends Common {
|
||||
|
||||
// We should store but not pre-fetch data for this transaction
|
||||
assertTrue(storageManager.canStoreData(transactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, transactionData).isPass());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user