mirror of
https://github.com/Qortal/qortal.git
synced 2025-04-14 23:35:54 +00:00
Merge remote-tracking branch 'kenny/master' into feature/search-keywords
This commit is contained in:
commit
1d79df078e
@ -2,7 +2,6 @@ package org.qortal.controller.arbitrary;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.qortal.api.resource.TransactionsResource;
|
||||
import org.qortal.controller.Controller;
|
||||
import org.qortal.data.arbitrary.ArbitraryResourceData;
|
||||
import org.qortal.data.transaction.ArbitraryTransactionData;
|
||||
@ -14,12 +13,16 @@ import org.qortal.repository.Repository;
|
||||
import org.qortal.repository.RepositoryManager;
|
||||
import org.qortal.settings.Settings;
|
||||
import org.qortal.transaction.ArbitraryTransaction;
|
||||
import org.qortal.transaction.Transaction;
|
||||
import org.qortal.utils.Base58;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class ArbitraryDataCacheManager extends Thread {
|
||||
|
||||
@ -86,47 +89,10 @@ public class ArbitraryDataCacheManager extends Thread {
|
||||
|
||||
// Update arbitrary resource caches
|
||||
try {
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updating resource cache, queue",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
|
||||
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
|
||||
arbitraryTransaction.updateArbitraryResourceCache(repository);
|
||||
arbitraryTransaction.updateArbitraryMetadataCache(repository);
|
||||
arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, new HashSet<>(0), new HashMap<>(0));
|
||||
repository.saveChanges();
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updated resource cache",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updating resource status, queue",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
|
||||
// Update status as separate commit, as this is more prone to failure
|
||||
arbitraryTransaction.updateArbitraryResourceStatus(repository);
|
||||
repository.saveChanges();
|
||||
@ -137,7 +103,7 @@ public class ArbitraryDataCacheManager extends Thread {
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updated resource status",
|
||||
"updated resource cache and status, queue",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
@ -201,64 +167,61 @@ public class ArbitraryDataCacheManager extends Thread {
|
||||
LOGGER.info("Building arbitrary resources cache...");
|
||||
SplashFrame.getInstance().updateStatus("Building QDN cache - please wait...");
|
||||
|
||||
final int batchSize = 100;
|
||||
final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
|
||||
int offset = 0;
|
||||
|
||||
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder
|
||||
= repository.getArbitraryRepository().getLatestArbitraryTransactions();
|
||||
|
||||
LOGGER.info("arbitrary transactions: count = " + allArbitraryTransactionsInDescendingOrder.size());
|
||||
|
||||
List<ArbitraryResourceData> resources = repository.getArbitraryRepository().getArbitraryResources(null, null, true);
|
||||
|
||||
Map<ArbitraryTransactionDataHashWrapper, ArbitraryResourceData> resourceByWrapper = new HashMap<>(resources.size());
|
||||
for( ArbitraryResourceData resource : resources ) {
|
||||
resourceByWrapper.put(
|
||||
new ArbitraryTransactionDataHashWrapper(resource.service.value, resource.name, resource.identifier),
|
||||
resource
|
||||
);
|
||||
}
|
||||
|
||||
LOGGER.info("arbitrary resources: count = " + resourceByWrapper.size());
|
||||
|
||||
Set<ArbitraryTransactionDataHashWrapper> latestTransactionsWrapped = new HashSet<>(allArbitraryTransactionsInDescendingOrder.size());
|
||||
|
||||
// Loop through all ARBITRARY transactions, and determine latest state
|
||||
while (!Controller.isStopping()) {
|
||||
LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1);
|
||||
|
||||
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false);
|
||||
if (signatures.isEmpty()) {
|
||||
List<ArbitraryTransactionData> transactionsToProcess
|
||||
= allArbitraryTransactionsInDescendingOrder.stream()
|
||||
.skip(offset)
|
||||
.limit(batchSize)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if (transactionsToProcess.isEmpty()) {
|
||||
// Complete
|
||||
break;
|
||||
}
|
||||
|
||||
// Expand signatures to transactions
|
||||
for (byte[] signature : signatures) {
|
||||
try {
|
||||
ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository
|
||||
.getTransactionRepository().fromSignature(signature);
|
||||
|
||||
try {
|
||||
for( ArbitraryTransactionData transactionData : transactionsToProcess) {
|
||||
if (transactionData.getService() == null) {
|
||||
// Unsupported service - ignore this resource
|
||||
continue;
|
||||
}
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updating resource cache, build",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
latestTransactionsWrapped.add(new ArbitraryTransactionDataHashWrapper(transactionData));
|
||||
|
||||
// Update arbitrary resource caches
|
||||
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
|
||||
arbitraryTransaction.updateArbitraryResourceCache(repository);
|
||||
arbitraryTransaction.updateArbitraryMetadataCache(repository);
|
||||
repository.saveChanges();
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updated resource cache",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
} catch (DataException e) {
|
||||
repository.discardChanges();
|
||||
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, latestTransactionsWrapped, resourceByWrapper);
|
||||
}
|
||||
repository.saveChanges();
|
||||
} catch (DataException e) {
|
||||
repository.discardChanges();
|
||||
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
}
|
||||
offset += batchSize;
|
||||
}
|
||||
@ -288,7 +251,7 @@ public class ArbitraryDataCacheManager extends Thread {
|
||||
LOGGER.info("Refreshing arbitrary resource statuses for locally hosted transactions...");
|
||||
SplashFrame.getInstance().updateStatus("Refreshing statuses - please wait...");
|
||||
|
||||
final int batchSize = 100;
|
||||
final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
|
||||
int offset = 0;
|
||||
|
||||
// Loop through all ARBITRARY transactions, and determine latest state
|
||||
@ -301,45 +264,21 @@ public class ArbitraryDataCacheManager extends Thread {
|
||||
break;
|
||||
}
|
||||
|
||||
// Loop through hosted transactions
|
||||
for (ArbitraryTransactionData transactionData : hostedTransactions) {
|
||||
|
||||
try {
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updating resource status",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
try {
|
||||
// Loop through hosted transactions
|
||||
for (ArbitraryTransactionData transactionData : hostedTransactions) {
|
||||
|
||||
// Determine status and update cache
|
||||
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
|
||||
arbitraryTransaction.updateArbitraryResourceStatus(repository);
|
||||
repository.saveChanges();
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
transactionData.getIdentifier(),
|
||||
transactionData.getName(),
|
||||
transactionData.getService().name(),
|
||||
"updated resource status",
|
||||
transactionData.getTimestamp(),
|
||||
transactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
|
||||
} catch (DataException e) {
|
||||
repository.discardChanges();
|
||||
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
}
|
||||
repository.saveChanges();
|
||||
} catch (DataException e) {
|
||||
repository.discardChanges();
|
||||
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
offset += batchSize;
|
||||
}
|
||||
|
||||
|
||||
@ -2,7 +2,6 @@ package org.qortal.controller.arbitrary;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
|
||||
import org.qortal.data.transaction.ArbitraryTransactionData;
|
||||
import org.qortal.data.transaction.TransactionData;
|
||||
import org.qortal.event.DataMonitorEvent;
|
||||
@ -23,9 +22,12 @@ import java.nio.file.Paths;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD;
|
||||
|
||||
@ -80,6 +82,19 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
final int limit = 100;
|
||||
int offset = 0;
|
||||
|
||||
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
|
||||
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
allArbitraryTransactionsInDescendingOrder
|
||||
= repository.getArbitraryRepository()
|
||||
.getLatestArbitraryTransactions();
|
||||
} catch( Exception e) {
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
|
||||
}
|
||||
|
||||
Set<ArbitraryTransactionData> processedTransactions = new HashSet<>();
|
||||
|
||||
try {
|
||||
while (!isStopping) {
|
||||
Thread.sleep(30000);
|
||||
@ -110,27 +125,31 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
|
||||
// Any arbitrary transactions we want to fetch data for?
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true);
|
||||
// LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
|
||||
List<ArbitraryTransactionData> transactions = allArbitraryTransactionsInDescendingOrder.stream().skip(offset).limit(limit).collect(Collectors.toList());
|
||||
if (isStopping) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (signatures == null || signatures.isEmpty()) {
|
||||
if (transactions == null || transactions.isEmpty()) {
|
||||
offset = 0;
|
||||
continue;
|
||||
allArbitraryTransactionsInDescendingOrder
|
||||
= repository.getArbitraryRepository()
|
||||
.getLatestArbitraryTransactions();
|
||||
transactions = allArbitraryTransactionsInDescendingOrder.stream().limit(limit).collect(Collectors.toList());
|
||||
processedTransactions.clear();
|
||||
}
|
||||
|
||||
offset += limit;
|
||||
now = NTP.getTime();
|
||||
|
||||
// Loop through the signatures in this batch
|
||||
for (int i=0; i<signatures.size(); i++) {
|
||||
for (int i=0; i<transactions.size(); i++) {
|
||||
if (isStopping) {
|
||||
return;
|
||||
}
|
||||
|
||||
byte[] signature = signatures.get(i);
|
||||
if (signature == null) {
|
||||
ArbitraryTransactionData arbitraryTransactionData = transactions.get(i);
|
||||
if (arbitraryTransactionData == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -139,9 +158,7 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
Thread.sleep(5000);
|
||||
}
|
||||
|
||||
// Fetch the transaction data
|
||||
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
|
||||
if (arbitraryTransactionData == null || arbitraryTransactionData.getService() == null) {
|
||||
if (arbitraryTransactionData.getService() == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -150,6 +167,8 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
continue;
|
||||
}
|
||||
|
||||
boolean mostRecentTransaction = processedTransactions.add(arbitraryTransactionData);
|
||||
|
||||
// Check if we have the complete file
|
||||
boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData);
|
||||
|
||||
@ -186,28 +205,38 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
}
|
||||
|
||||
// Check to see if we have had a more recent PUT
|
||||
Optional<ArbitraryTransactionData> moreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
|
||||
boolean hasMoreRecentPutTransaction = moreRecentPutTransaction.isPresent();
|
||||
if (hasMoreRecentPutTransaction) {
|
||||
if (!mostRecentTransaction) {
|
||||
// There is a more recent PUT transaction than the one we are currently processing.
|
||||
// When a PUT is issued, it replaces any layers that would have been there before.
|
||||
// Therefore any data relating to this older transaction is no longer needed.
|
||||
LOGGER.info(String.format("Newer PUT found for %s %s since transaction %s. " +
|
||||
"Deleting all files associated with the earlier transaction.", arbitraryTransactionData.getService(),
|
||||
arbitraryTransactionData.getName(), Base58.encode(signature)));
|
||||
arbitraryTransactionData.getName(), Base58.encode(arbitraryTransactionData.getSignature())));
|
||||
|
||||
ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"deleting data due to replacement",
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
moreRecentPutTransaction.get().getTimestamp()
|
||||
)
|
||||
);
|
||||
|
||||
Optional<ArbitraryTransactionData> moreRecentPutTransaction
|
||||
= processedTransactions.stream()
|
||||
.filter(data -> data.equals(arbitraryTransactionData))
|
||||
.findAny();
|
||||
|
||||
if( moreRecentPutTransaction.isPresent() ) {
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"deleting data due to replacement",
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
moreRecentPutTransaction.get().getTimestamp()
|
||||
)
|
||||
);
|
||||
}
|
||||
else {
|
||||
LOGGER.warn("Something went wrong with the most recent put transaction determination!");
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -226,18 +255,21 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
LOGGER.debug(String.format("Transaction %s has complete file and all chunks",
|
||||
Base58.encode(arbitraryTransactionData.getSignature())));
|
||||
|
||||
ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"deleting data due to age",
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
arbitraryTransactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
boolean wasDeleted = ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
|
||||
|
||||
if( wasDeleted ) {
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"deleting file, retaining chunks",
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
arbitraryTransactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -452,18 +484,6 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
// Relates to a different name - don't delete it
|
||||
return false;
|
||||
}
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"randomly selected for deletion",
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
arbitraryTransactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
} catch (DataException e) {
|
||||
@ -473,6 +493,7 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
}
|
||||
|
||||
LOGGER.info("Deleting random file {} because we have reached max storage capacity...", randomItem.toString());
|
||||
fireRandomItemDeletionNotification(randomItem, repository, "Deleting random file, because we have reached max storage capacity");
|
||||
boolean success = randomItem.delete();
|
||||
if (success) {
|
||||
try {
|
||||
@ -487,6 +508,35 @@ public class ArbitraryDataCleanupManager extends Thread {
|
||||
return false;
|
||||
}
|
||||
|
||||
private void fireRandomItemDeletionNotification(File randomItem, Repository repository, String reason) {
|
||||
try {
|
||||
Path parentFileNamePath = randomItem.toPath().toAbsolutePath().getParent().getFileName();
|
||||
if (parentFileNamePath != null) {
|
||||
String signature58 = parentFileNamePath.toString();
|
||||
byte[] signature = Base58.decode(signature58);
|
||||
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
|
||||
if (transactionData != null && transactionData.getType() == Transaction.TransactionType.ARBITRARY) {
|
||||
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
reason,
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
arbitraryTransactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanupTempDirectory(String folder, long now, long minAge) {
|
||||
String baseDir = Settings.getInstance().getTempDataPath();
|
||||
Path tempDir = Paths.get(baseDir, folder);
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
package org.qortal.controller.arbitrary;
|
||||
|
||||
public class ArbitraryDataExamination {
|
||||
|
||||
private boolean pass;
|
||||
|
||||
private String notes;
|
||||
|
||||
public ArbitraryDataExamination(boolean pass, String notes) {
|
||||
this.pass = pass;
|
||||
this.notes = notes;
|
||||
}
|
||||
|
||||
public boolean isPass() {
|
||||
return pass;
|
||||
}
|
||||
|
||||
public String getNotes() {
|
||||
return notes;
|
||||
}
|
||||
}
|
||||
@ -198,13 +198,35 @@ public class ArbitraryDataManager extends Thread {
|
||||
final int limit = 100;
|
||||
int offset = 0;
|
||||
|
||||
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
|
||||
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
|
||||
if( name == null ) {
|
||||
allArbitraryTransactionsInDescendingOrder
|
||||
= repository.getArbitraryRepository()
|
||||
.getLatestArbitraryTransactions();
|
||||
}
|
||||
else {
|
||||
allArbitraryTransactionsInDescendingOrder
|
||||
= repository.getArbitraryRepository()
|
||||
.getLatestArbitraryTransactionsByName(name);
|
||||
}
|
||||
} catch( Exception e) {
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
|
||||
}
|
||||
|
||||
// collect processed transactions in a set to ensure outdated data transactions do not get fetched
|
||||
Set<ArbitraryTransactionDataHashWrapper> processedTransactions = new HashSet<>();
|
||||
|
||||
while (!isStopping) {
|
||||
Thread.sleep(1000L);
|
||||
|
||||
// Any arbitrary transactions we want to fetch data for?
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, name, null, ConfirmationStatus.BOTH, limit, offset, true);
|
||||
// LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
|
||||
List<byte[]> signatures = processTransactionsForSignatures(limit, offset, allArbitraryTransactionsInDescendingOrder, processedTransactions);
|
||||
|
||||
if (signatures == null || signatures.isEmpty()) {
|
||||
offset = 0;
|
||||
break;
|
||||
@ -226,7 +248,8 @@ public class ArbitraryDataManager extends Thread {
|
||||
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
|
||||
|
||||
// Skip transactions that we don't need to proactively store data for
|
||||
if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) {
|
||||
ArbitraryDataExamination arbitraryDataExamination = storageManager.shouldPreFetchData(repository, arbitraryTransactionData);
|
||||
if (!arbitraryDataExamination.isPass()) {
|
||||
iterator.remove();
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
@ -235,7 +258,7 @@ public class ArbitraryDataManager extends Thread {
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"don't need to proactively store, skipping",
|
||||
arbitraryDataExamination.getNotes(),
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
arbitraryTransactionData.getTimestamp()
|
||||
)
|
||||
@ -342,7 +365,7 @@ public class ArbitraryDataManager extends Thread {
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
allArbitraryTransactionsInDescendingOrder
|
||||
= repository.getArbitraryRepository()
|
||||
.getLatestArbitraryTransactions(Settings.getInstance().getDataFetchLimit());
|
||||
.getLatestArbitraryTransactions();
|
||||
} catch( Exception e) {
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
|
||||
@ -410,18 +433,6 @@ public class ArbitraryDataManager extends Thread {
|
||||
// fetch the metadata and notify the event bus again
|
||||
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
|
||||
|
||||
EventBus.INSTANCE.notify(
|
||||
new DataMonitorEvent(
|
||||
System.currentTimeMillis(),
|
||||
arbitraryTransactionData.getIdentifier(),
|
||||
arbitraryTransactionData.getName(),
|
||||
arbitraryTransactionData.getService().name(),
|
||||
"fetching metadata",
|
||||
arbitraryTransactionData.getTimestamp(),
|
||||
arbitraryTransactionData.getTimestamp()
|
||||
)
|
||||
);
|
||||
|
||||
// Ask our connected peers if they have metadata for this signature
|
||||
fetchMetadata(arbitraryTransactionData);
|
||||
|
||||
@ -444,10 +455,14 @@ public class ArbitraryDataManager extends Thread {
|
||||
}
|
||||
}
|
||||
|
||||
private static List<byte[]> processTransactionsForSignatures(int limit, int offset, List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder, Set<ArbitraryTransactionDataHashWrapper> processedTransactions) {
|
||||
private static List<byte[]> processTransactionsForSignatures(
|
||||
int limit,
|
||||
int offset,
|
||||
List<ArbitraryTransactionData> transactionsInDescendingOrder,
|
||||
Set<ArbitraryTransactionDataHashWrapper> processedTransactions) {
|
||||
// these transactions are in descending order, latest transactions come first
|
||||
List<ArbitraryTransactionData> transactions
|
||||
= allArbitraryTransactionsInDescendingOrder.stream()
|
||||
= transactionsInDescendingOrder.stream()
|
||||
.skip(offset)
|
||||
.limit(limit)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
@ -155,31 +155,31 @@ public class ArbitraryDataStorageManager extends Thread {
|
||||
* @param arbitraryTransactionData - the transaction
|
||||
* @return boolean - whether to prefetch or not
|
||||
*/
|
||||
public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
|
||||
public ArbitraryDataExamination shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
|
||||
String name = arbitraryTransactionData.getName();
|
||||
|
||||
// Only fetch data associated with hashes, as we already have RAW_DATA
|
||||
if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) {
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false, "Only fetch data associated with hashes");
|
||||
}
|
||||
|
||||
// Don't fetch anything more if we're (nearly) out of space
|
||||
// Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
|
||||
// avoid a fetch/delete loop
|
||||
if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) {
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false,"Don't fetch anything more if we're (nearly) out of space");
|
||||
}
|
||||
|
||||
// Don't fetch anything if we're (nearly) out of space for this name
|
||||
// Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
|
||||
// avoid a fetch/delete loop
|
||||
if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) {
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false, "Don't fetch anything if we're (nearly) out of space for this name");
|
||||
}
|
||||
|
||||
// Don't store data unless it's an allowed type (public/private)
|
||||
if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false, "Don't store data unless it's an allowed type (public/private)");
|
||||
}
|
||||
|
||||
// Handle transactions without names differently
|
||||
@ -189,21 +189,21 @@ public class ArbitraryDataStorageManager extends Thread {
|
||||
|
||||
// Never fetch data from blocked names, even if they are followed
|
||||
if (ListUtils.isNameBlocked(name)) {
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false, "blocked name");
|
||||
}
|
||||
|
||||
switch (Settings.getInstance().getStoragePolicy()) {
|
||||
case FOLLOWED:
|
||||
case FOLLOWED_OR_VIEWED:
|
||||
return ListUtils.isFollowingName(name);
|
||||
return new ArbitraryDataExamination(ListUtils.isFollowingName(name), Settings.getInstance().getStoragePolicy().name());
|
||||
|
||||
case ALL:
|
||||
return true;
|
||||
return new ArbitraryDataExamination(true, Settings.getInstance().getStoragePolicy().name());
|
||||
|
||||
case NONE:
|
||||
case VIEWED:
|
||||
default:
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name());
|
||||
}
|
||||
}
|
||||
|
||||
@ -214,17 +214,17 @@ public class ArbitraryDataStorageManager extends Thread {
|
||||
*
|
||||
* @return boolean - whether the storage policy allows for unnamed data
|
||||
*/
|
||||
private boolean shouldPreFetchDataWithoutName() {
|
||||
private ArbitraryDataExamination shouldPreFetchDataWithoutName() {
|
||||
switch (Settings.getInstance().getStoragePolicy()) {
|
||||
case ALL:
|
||||
return true;
|
||||
return new ArbitraryDataExamination(true, "Fetching all data");
|
||||
|
||||
case NONE:
|
||||
case VIEWED:
|
||||
case FOLLOWED:
|
||||
case FOLLOWED_OR_VIEWED:
|
||||
default:
|
||||
return false;
|
||||
return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ import java.util.Objects;
|
||||
|
||||
public class ArbitraryTransactionDataHashWrapper {
|
||||
|
||||
private final ArbitraryTransactionData data;
|
||||
private ArbitraryTransactionData data;
|
||||
|
||||
private int service;
|
||||
|
||||
@ -23,6 +23,12 @@ public class ArbitraryTransactionDataHashWrapper {
|
||||
this.identifier = data.getIdentifier();
|
||||
}
|
||||
|
||||
public ArbitraryTransactionDataHashWrapper(int service, String name, String identifier) {
|
||||
this.service = service;
|
||||
this.name = name;
|
||||
this.identifier = identifier;
|
||||
}
|
||||
|
||||
public ArbitraryTransactionData getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
@ -27,7 +27,9 @@ public interface ArbitraryRepository {
|
||||
|
||||
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
|
||||
|
||||
List<ArbitraryTransactionData> getLatestArbitraryTransactions(int limit) throws DataException;
|
||||
List<ArbitraryTransactionData> getLatestArbitraryTransactions() throws DataException;
|
||||
|
||||
List<ArbitraryTransactionData> getLatestArbitraryTransactionsByName(String name) throws DataException;
|
||||
|
||||
public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException;
|
||||
|
||||
|
||||
@ -228,18 +228,86 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ArbitraryTransactionData> getLatestArbitraryTransactions(int limit) throws DataException {
|
||||
public List<ArbitraryTransactionData> getLatestArbitraryTransactions() throws DataException {
|
||||
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
|
||||
"tx_group_id, block_height, approval_status, approval_height, " +
|
||||
"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
|
||||
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
|
||||
"JOIN Transactions USING (signature) " +
|
||||
"WHERE name IS NOT NULL " +
|
||||
"ORDER BY created_when DESC " +
|
||||
"LIMIT ?";
|
||||
"ORDER BY created_when DESC";
|
||||
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
|
||||
|
||||
try (ResultSet resultSet = this.repository.checkedExecute(sql, limit)) {
|
||||
try (ResultSet resultSet = this.repository.checkedExecute(sql)) {
|
||||
if (resultSet == null)
|
||||
return new ArrayList<>(0);
|
||||
|
||||
do {
|
||||
byte[] reference = resultSet.getBytes(2);
|
||||
byte[] signature = resultSet.getBytes(3);
|
||||
byte[] creatorPublicKey = resultSet.getBytes(4);
|
||||
long timestamp = resultSet.getLong(5);
|
||||
|
||||
Long fee = resultSet.getLong(6);
|
||||
if (fee == 0 && resultSet.wasNull())
|
||||
fee = null;
|
||||
|
||||
int txGroupId = resultSet.getInt(7);
|
||||
|
||||
Integer blockHeight = resultSet.getInt(8);
|
||||
if (blockHeight == 0 && resultSet.wasNull())
|
||||
blockHeight = null;
|
||||
|
||||
ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
|
||||
Integer approvalHeight = resultSet.getInt(10);
|
||||
if (approvalHeight == 0 && resultSet.wasNull())
|
||||
approvalHeight = null;
|
||||
|
||||
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
|
||||
|
||||
int version = resultSet.getInt(11);
|
||||
int nonce = resultSet.getInt(12);
|
||||
int serviceInt = resultSet.getInt(13);
|
||||
int size = resultSet.getInt(14);
|
||||
boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
|
||||
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
|
||||
byte[] data = resultSet.getBytes(16);
|
||||
byte[] metadataHash = resultSet.getBytes(17);
|
||||
String nameResult = resultSet.getString(18);
|
||||
String identifierResult = resultSet.getString(19);
|
||||
Method method = Method.valueOf(resultSet.getInt(20));
|
||||
byte[] secret = resultSet.getBytes(21);
|
||||
Compression compression = Compression.valueOf(resultSet.getInt(22));
|
||||
// FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls.
|
||||
|
||||
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
|
||||
version, serviceInt, nonce, size, nameResult, identifierResult, method, secret,
|
||||
compression, data, dataType, metadataHash, null);
|
||||
|
||||
arbitraryTransactionData.add(transactionData);
|
||||
} while (resultSet.next());
|
||||
|
||||
return arbitraryTransactionData;
|
||||
} catch (SQLException e) {
|
||||
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
|
||||
} catch (Exception e) {
|
||||
LOGGER.error(e.getMessage(), e);
|
||||
return new ArrayList<>(0);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<ArbitraryTransactionData> getLatestArbitraryTransactionsByName( String name ) throws DataException {
|
||||
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
|
||||
"tx_group_id, block_height, approval_status, approval_height, " +
|
||||
"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
|
||||
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
|
||||
"JOIN Transactions USING (signature) " +
|
||||
"WHERE name = ? " +
|
||||
"ORDER BY created_when DESC";
|
||||
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
|
||||
|
||||
try (ResultSet resultSet = this.repository.checkedExecute(sql, name)) {
|
||||
if (resultSet == null)
|
||||
return new ArrayList<>(0);
|
||||
|
||||
|
||||
@ -508,9 +508,9 @@ public class Settings {
|
||||
*/
|
||||
private boolean connectionPoolMonitorEnabled = false;
|
||||
|
||||
private int dataFetchLimit = 1_000_000;
|
||||
private int buildArbitraryResourcesBatchSize = 200;
|
||||
|
||||
// Domain mapping
|
||||
// Domain mapping
|
||||
public static class ThreadLimit {
|
||||
private String messageType;
|
||||
private Integer limit;
|
||||
@ -1336,7 +1336,7 @@ public class Settings {
|
||||
return connectionPoolMonitorEnabled;
|
||||
}
|
||||
|
||||
public int getDataFetchLimit() {
|
||||
return dataFetchLimit;
|
||||
public int getBuildArbitraryResourcesBatchSize() {
|
||||
return buildArbitraryResourcesBatchSize;
|
||||
}
|
||||
}
|
||||
|
||||
@ -9,6 +9,7 @@ import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
|
||||
import org.qortal.arbitrary.misc.Service;
|
||||
import org.qortal.block.BlockChain;
|
||||
import org.qortal.controller.arbitrary.ArbitraryDataManager;
|
||||
import org.qortal.controller.arbitrary.ArbitraryTransactionDataHashWrapper;
|
||||
import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
|
||||
import org.qortal.crypto.Crypto;
|
||||
import org.qortal.crypto.MemoryPoW;
|
||||
@ -31,8 +32,12 @@ import org.qortal.utils.ArbitraryTransactionUtils;
|
||||
import org.qortal.utils.NTP;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class ArbitraryTransaction extends Transaction {
|
||||
@ -303,8 +308,7 @@ public class ArbitraryTransaction extends Transaction {
|
||||
// Add/update arbitrary resource caches, but don't update the status as this involves time-consuming
|
||||
// disk reads, and is more prone to failure. The status will be updated on metadata retrieval, or when
|
||||
// accessing the resource.
|
||||
this.updateArbitraryResourceCache(repository);
|
||||
this.updateArbitraryMetadataCache(repository);
|
||||
this.updateArbitraryResourceCacheIncludingMetadata(repository, new HashSet<>(0), new HashMap<>(0));
|
||||
|
||||
repository.saveChanges();
|
||||
|
||||
@ -360,7 +364,10 @@ public class ArbitraryTransaction extends Transaction {
|
||||
*
|
||||
* @throws DataException
|
||||
*/
|
||||
public void updateArbitraryResourceCache(Repository repository) throws DataException {
|
||||
public void updateArbitraryResourceCacheIncludingMetadata(
|
||||
Repository repository,
|
||||
Set<ArbitraryTransactionDataHashWrapper> latestTransactionWrappers,
|
||||
Map<ArbitraryTransactionDataHashWrapper, ArbitraryResourceData> resourceByWrapper) throws DataException {
|
||||
// Don't cache resources without a name (such as auto updates)
|
||||
if (arbitraryTransactionData.getName() == null) {
|
||||
return;
|
||||
@ -385,29 +392,42 @@ public class ArbitraryTransaction extends Transaction {
|
||||
arbitraryResourceData.name = name;
|
||||
arbitraryResourceData.identifier = identifier;
|
||||
|
||||
// Get the latest transaction
|
||||
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
|
||||
if (latestTransactionData == null) {
|
||||
LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData);
|
||||
// We don't have a latest transaction, so delete from cache
|
||||
repository.getArbitraryRepository().delete(arbitraryResourceData);
|
||||
return;
|
||||
final ArbitraryTransactionDataHashWrapper wrapper = new ArbitraryTransactionDataHashWrapper(arbitraryTransactionData);
|
||||
|
||||
ArbitraryTransactionData latestTransactionData;
|
||||
if( latestTransactionWrappers.contains(wrapper)) {
|
||||
latestTransactionData
|
||||
= latestTransactionWrappers.stream()
|
||||
.filter( latestWrapper -> latestWrapper.equals(wrapper))
|
||||
.findAny().get()
|
||||
.getData();
|
||||
}
|
||||
else {
|
||||
// Get the latest transaction
|
||||
latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
|
||||
if (latestTransactionData == null) {
|
||||
LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData);
|
||||
// We don't have a latest transaction, so delete from cache
|
||||
repository.getArbitraryRepository().delete(arbitraryResourceData);
|
||||
return;
|
||||
}
|
||||
}
|
||||
ArbitraryResourceData existingArbitraryResourceData = resourceByWrapper.get(wrapper);
|
||||
|
||||
// Get existing cached entry if it exists
|
||||
ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository()
|
||||
.getArbitraryResource(service, name, identifier);
|
||||
|
||||
LOGGER.info("updating existing arbitraryResourceData" + existingArbitraryResourceData);
|
||||
if( existingArbitraryResourceData == null ) {
|
||||
// Get existing cached entry if it exists
|
||||
existingArbitraryResourceData = repository.getArbitraryRepository()
|
||||
.getArbitraryResource(service, name, identifier);
|
||||
}
|
||||
|
||||
// Check for existing cached data
|
||||
if (existingArbitraryResourceData == null) {
|
||||
// Nothing exists yet, so set creation date from the current transaction (it will be reduced later if needed)
|
||||
arbitraryResourceData.created = arbitraryTransactionData.getTimestamp();
|
||||
arbitraryResourceData.updated = null;
|
||||
LOGGER.info("updated = null, reason = existingArbitraryResourceData == null" );
|
||||
}
|
||||
else {
|
||||
resourceByWrapper.put(wrapper, existingArbitraryResourceData);
|
||||
// An entry already exists - update created time from current transaction if this is older
|
||||
arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp());
|
||||
|
||||
@ -415,22 +435,44 @@ public class ArbitraryTransaction extends Transaction {
|
||||
if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) {
|
||||
// Latest transaction matches created time, so it hasn't been updated
|
||||
arbitraryResourceData.updated = null;
|
||||
LOGGER.info(
|
||||
"updated = null, reason: existingArbitraryResourceData.created == latestTransactionData.getTimestamp() == " +
|
||||
existingArbitraryResourceData.created );
|
||||
}
|
||||
else {
|
||||
arbitraryResourceData.updated = latestTransactionData.getTimestamp();
|
||||
LOGGER.info("setting updated to a non-null value");
|
||||
}
|
||||
}
|
||||
|
||||
arbitraryResourceData.size = latestTransactionData.getSize();
|
||||
|
||||
LOGGER.info("saving updated arbitraryResourceData: updated = " + arbitraryResourceData.updated);
|
||||
|
||||
// Save
|
||||
repository.getArbitraryRepository().save(arbitraryResourceData);
|
||||
|
||||
// Update metadata for latest transaction if it is local
|
||||
if (latestTransactionData.getMetadataHash() != null) {
|
||||
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
|
||||
if (metadataFile.exists()) {
|
||||
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
|
||||
try {
|
||||
transactionMetadata.read();
|
||||
|
||||
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
|
||||
metadata.setArbitraryResourceData(arbitraryResourceData);
|
||||
metadata.setTitle(transactionMetadata.getTitle());
|
||||
metadata.setDescription(transactionMetadata.getDescription());
|
||||
metadata.setCategory(transactionMetadata.getCategory());
|
||||
metadata.setTags(transactionMetadata.getTags());
|
||||
repository.getArbitraryRepository().save(metadata);
|
||||
|
||||
} catch (IOException e) {
|
||||
// Ignore, as we can add it again later
|
||||
}
|
||||
} else {
|
||||
// We don't have a local copy of this metadata file, so delete it from the cache
|
||||
// It will be re-added if the file later arrives via the network
|
||||
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
|
||||
metadata.setArbitraryResourceData(arbitraryResourceData);
|
||||
repository.getArbitraryRepository().delete(metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void updateArbitraryResourceStatus(Repository repository) throws DataException {
|
||||
@ -465,60 +507,4 @@ public class ArbitraryTransaction extends Transaction {
|
||||
repository.getArbitraryRepository().setStatus(arbitraryResourceData, status);
|
||||
}
|
||||
|
||||
public void updateArbitraryMetadataCache(Repository repository) throws DataException {
|
||||
// Get the latest transaction
|
||||
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
|
||||
if (latestTransactionData == null) {
|
||||
// We don't have a latest transaction, so give up
|
||||
return;
|
||||
}
|
||||
|
||||
Service service = latestTransactionData.getService();
|
||||
String name = latestTransactionData.getName();
|
||||
String identifier = latestTransactionData.getIdentifier();
|
||||
|
||||
if (service == null) {
|
||||
// Unsupported service - ignore this resource
|
||||
return;
|
||||
}
|
||||
|
||||
// In the cache we store null identifiers as "default", as it is part of the primary key
|
||||
if (identifier == null) {
|
||||
identifier = "default";
|
||||
}
|
||||
|
||||
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
|
||||
arbitraryResourceData.service = service;
|
||||
arbitraryResourceData.name = name;
|
||||
arbitraryResourceData.identifier = identifier;
|
||||
|
||||
// Update metadata for latest transaction if it is local
|
||||
if (latestTransactionData.getMetadataHash() != null) {
|
||||
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
|
||||
if (metadataFile.exists()) {
|
||||
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
|
||||
try {
|
||||
transactionMetadata.read();
|
||||
|
||||
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
|
||||
metadata.setArbitraryResourceData(arbitraryResourceData);
|
||||
metadata.setTitle(transactionMetadata.getTitle());
|
||||
metadata.setDescription(transactionMetadata.getDescription());
|
||||
metadata.setCategory(transactionMetadata.getCategory());
|
||||
metadata.setTags(transactionMetadata.getTags());
|
||||
repository.getArbitraryRepository().save(metadata);
|
||||
|
||||
} catch (IOException e) {
|
||||
// Ignore, as we can add it again later
|
||||
}
|
||||
} else {
|
||||
// We don't have a local copy of this metadata file, so delete it from the cache
|
||||
// It will be re-added if the file later arrives via the network
|
||||
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
|
||||
metadata.setArbitraryResourceData(arbitraryResourceData);
|
||||
repository.getArbitraryRepository().delete(metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -209,7 +209,15 @@ public class ArbitraryTransactionUtils {
|
||||
return ArbitraryTransactionUtils.isFileRecent(filePath, now, cleanupAfter);
|
||||
}
|
||||
|
||||
public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
|
||||
/**
|
||||
*
|
||||
* @param arbitraryTransactionData
|
||||
* @param now
|
||||
* @param cleanupAfter
|
||||
* @return true if file is deleted, otherwise return false
|
||||
* @throws DataException
|
||||
*/
|
||||
public static boolean deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
|
||||
byte[] completeHash = arbitraryTransactionData.getData();
|
||||
byte[] signature = arbitraryTransactionData.getSignature();
|
||||
|
||||
@ -220,6 +228,11 @@ public class ArbitraryTransactionUtils {
|
||||
"if needed", Base58.encode(completeHash));
|
||||
|
||||
arbitraryDataFile.delete();
|
||||
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -73,14 +73,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
|
||||
// We should store and pre-fetch data for this transaction
|
||||
assertEquals(StoragePolicy.FOLLOWED_OR_VIEWED, Settings.getInstance().getStoragePolicy());
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
|
||||
// Now unfollow the name
|
||||
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
|
||||
|
||||
// We should store but not pre-fetch data for this transaction
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
}
|
||||
}
|
||||
|
||||
@ -108,14 +108,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
|
||||
// We should store and pre-fetch data for this transaction
|
||||
assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy());
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
|
||||
// Now unfollow the name
|
||||
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
|
||||
|
||||
// We shouldn't store or pre-fetch data for this transaction
|
||||
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
}
|
||||
}
|
||||
|
||||
@ -143,14 +143,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
|
||||
// We should store but not pre-fetch data for this transaction
|
||||
assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy());
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
|
||||
// Now unfollow the name
|
||||
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
|
||||
|
||||
// We should store but not pre-fetch data for this transaction
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
}
|
||||
}
|
||||
|
||||
@ -178,14 +178,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
|
||||
// We should store and pre-fetch data for this transaction
|
||||
assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy());
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
|
||||
// Now unfollow the name
|
||||
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
|
||||
|
||||
// We should store and pre-fetch data for this transaction
|
||||
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
}
|
||||
}
|
||||
|
||||
@ -213,14 +213,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
|
||||
// We shouldn't store or pre-fetch data for this transaction
|
||||
assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy());
|
||||
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
|
||||
// Now unfollow the name
|
||||
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
|
||||
|
||||
// We shouldn't store or pre-fetch data for this transaction
|
||||
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
|
||||
}
|
||||
}
|
||||
|
||||
@ -236,7 +236,7 @@ public class ArbitraryDataStoragePolicyTests extends Common {
|
||||
|
||||
// We should store but not pre-fetch data for this transaction
|
||||
assertTrue(storageManager.canStoreData(transactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
|
||||
assertFalse(storageManager.shouldPreFetchData(repository, transactionData).isPass());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user