diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java index c30e190c..62d91109 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java @@ -2,7 +2,6 @@ package org.qortal.controller.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.api.resource.TransactionsResource; import org.qortal.controller.Controller; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.transaction.ArbitraryTransactionData; @@ -14,12 +13,16 @@ import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.transaction.ArbitraryTransaction; -import org.qortal.transaction.Transaction; import org.qortal.utils.Base58; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; public class ArbitraryDataCacheManager extends Thread { @@ -86,47 +89,10 @@ public class ArbitraryDataCacheManager extends Thread { // Update arbitrary resource caches try { - EventBus.INSTANCE.notify( - new DataMonitorEvent( - System.currentTimeMillis(), - transactionData.getIdentifier(), - transactionData.getName(), - transactionData.getService().name(), - "updating resource cache, queue", - transactionData.getTimestamp(), - transactionData.getTimestamp() - ) - ); - ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); - arbitraryTransaction.updateArbitraryResourceCache(repository); - arbitraryTransaction.updateArbitraryMetadataCache(repository); + arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, new HashSet<>(0), new HashMap<>(0)); repository.saveChanges(); - EventBus.INSTANCE.notify( - new DataMonitorEvent( - System.currentTimeMillis(), - transactionData.getIdentifier(), - transactionData.getName(), - transactionData.getService().name(), - "updated resource cache", - transactionData.getTimestamp(), - transactionData.getTimestamp() - ) - ); - - EventBus.INSTANCE.notify( - new DataMonitorEvent( - System.currentTimeMillis(), - transactionData.getIdentifier(), - transactionData.getName(), - transactionData.getService().name(), - "updating resource status, queue", - transactionData.getTimestamp(), - transactionData.getTimestamp() - ) - ); - // Update status as separate commit, as this is more prone to failure arbitraryTransaction.updateArbitraryResourceStatus(repository); repository.saveChanges(); @@ -137,7 +103,7 @@ public class ArbitraryDataCacheManager extends Thread { transactionData.getIdentifier(), transactionData.getName(), transactionData.getService().name(), - "updated resource status", + "updated resource cache and status, queue", transactionData.getTimestamp(), transactionData.getTimestamp() ) @@ -201,64 +167,61 @@ public class ArbitraryDataCacheManager extends Thread { LOGGER.info("Building arbitrary resources cache..."); SplashFrame.getInstance().updateStatus("Building QDN cache - please wait..."); - final int batchSize = 100; + final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize(); int offset = 0; + List allArbitraryTransactionsInDescendingOrder + = repository.getArbitraryRepository().getLatestArbitraryTransactions(); + + LOGGER.info("arbitrary transactions: count = " + allArbitraryTransactionsInDescendingOrder.size()); + + List resources = repository.getArbitraryRepository().getArbitraryResources(null, null, true); + + Map resourceByWrapper = new HashMap<>(resources.size()); + for( ArbitraryResourceData resource : resources ) { + resourceByWrapper.put( + new ArbitraryTransactionDataHashWrapper(resource.service.value, resource.name, resource.identifier), + resource + ); + } + + LOGGER.info("arbitrary resources: count = " + resourceByWrapper.size()); + + Set latestTransactionsWrapped = new HashSet<>(allArbitraryTransactionsInDescendingOrder.size()); + // Loop through all ARBITRARY transactions, and determine latest state while (!Controller.isStopping()) { LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1); - List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false); - if (signatures.isEmpty()) { + List transactionsToProcess + = allArbitraryTransactionsInDescendingOrder.stream() + .skip(offset) + .limit(batchSize) + .collect(Collectors.toList()); + + if (transactionsToProcess.isEmpty()) { // Complete break; } - // Expand signatures to transactions - for (byte[] signature : signatures) { - try { - ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository - .getTransactionRepository().fromSignature(signature); - + try { + for( ArbitraryTransactionData transactionData : transactionsToProcess) { if (transactionData.getService() == null) { // Unsupported service - ignore this resource continue; } - EventBus.INSTANCE.notify( - new DataMonitorEvent( - System.currentTimeMillis(), - transactionData.getIdentifier(), - transactionData.getName(), - transactionData.getService().name(), - "updating resource cache, build", - transactionData.getTimestamp(), - transactionData.getTimestamp() - ) - ); + latestTransactionsWrapped.add(new ArbitraryTransactionDataHashWrapper(transactionData)); // Update arbitrary resource caches ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); - arbitraryTransaction.updateArbitraryResourceCache(repository); - arbitraryTransaction.updateArbitraryMetadataCache(repository); - repository.saveChanges(); - - EventBus.INSTANCE.notify( - new DataMonitorEvent( - System.currentTimeMillis(), - transactionData.getIdentifier(), - transactionData.getName(), - transactionData.getService().name(), - "updated resource cache", - transactionData.getTimestamp(), - transactionData.getTimestamp() - ) - ); - } catch (DataException e) { - repository.discardChanges(); - - LOGGER.error(e.getMessage(), e); + arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, latestTransactionsWrapped, resourceByWrapper); } + repository.saveChanges(); + } catch (DataException e) { + repository.discardChanges(); + + LOGGER.error(e.getMessage(), e); } offset += batchSize; } @@ -288,7 +251,7 @@ public class ArbitraryDataCacheManager extends Thread { LOGGER.info("Refreshing arbitrary resource statuses for locally hosted transactions..."); SplashFrame.getInstance().updateStatus("Refreshing statuses - please wait..."); - final int batchSize = 100; + final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize(); int offset = 0; // Loop through all ARBITRARY transactions, and determine latest state @@ -301,45 +264,21 @@ public class ArbitraryDataCacheManager extends Thread { break; } - // Loop through hosted transactions - for (ArbitraryTransactionData transactionData : hostedTransactions) { - - try { - EventBus.INSTANCE.notify( - new DataMonitorEvent( - System.currentTimeMillis(), - transactionData.getIdentifier(), - transactionData.getName(), - transactionData.getService().name(), - "updating resource status", - transactionData.getTimestamp(), - transactionData.getTimestamp() - ) - ); + try { + // Loop through hosted transactions + for (ArbitraryTransactionData transactionData : hostedTransactions) { // Determine status and update cache ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); arbitraryTransaction.updateArbitraryResourceStatus(repository); - repository.saveChanges(); - - EventBus.INSTANCE.notify( - new DataMonitorEvent( - System.currentTimeMillis(), - transactionData.getIdentifier(), - transactionData.getName(), - transactionData.getService().name(), - "updated resource status", - transactionData.getTimestamp(), - transactionData.getTimestamp() - ) - ); - - } catch (DataException e) { - repository.discardChanges(); - - LOGGER.error(e.getMessage(), e); } + repository.saveChanges(); + } catch (DataException e) { + repository.discardChanges(); + + LOGGER.error(e.getMessage(), e); } + offset += batchSize; } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index 99044988..bc92395d 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -2,7 +2,6 @@ package org.qortal.controller.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.event.DataMonitorEvent; @@ -23,9 +22,12 @@ import java.nio.file.Paths; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD; @@ -80,6 +82,19 @@ public class ArbitraryDataCleanupManager extends Thread { final int limit = 100; int offset = 0; + List allArbitraryTransactionsInDescendingOrder; + + try (final Repository repository = RepositoryManager.getRepository()) { + allArbitraryTransactionsInDescendingOrder + = repository.getArbitraryRepository() + .getLatestArbitraryTransactions(); + } catch( Exception e) { + LOGGER.error(e.getMessage(), e); + allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0); + } + + Set processedTransactions = new HashSet<>(); + try { while (!isStopping) { Thread.sleep(30000); @@ -110,27 +125,31 @@ public class ArbitraryDataCleanupManager extends Thread { // Any arbitrary transactions we want to fetch data for? try (final Repository repository = RepositoryManager.getRepository()) { - List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true); - // LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit); + List transactions = allArbitraryTransactionsInDescendingOrder.stream().skip(offset).limit(limit).collect(Collectors.toList()); if (isStopping) { return; } - if (signatures == null || signatures.isEmpty()) { + if (transactions == null || transactions.isEmpty()) { offset = 0; - continue; + allArbitraryTransactionsInDescendingOrder + = repository.getArbitraryRepository() + .getLatestArbitraryTransactions(); + transactions = allArbitraryTransactionsInDescendingOrder.stream().limit(limit).collect(Collectors.toList()); + processedTransactions.clear(); } + offset += limit; now = NTP.getTime(); // Loop through the signatures in this batch - for (int i=0; i moreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData); - boolean hasMoreRecentPutTransaction = moreRecentPutTransaction.isPresent(); - if (hasMoreRecentPutTransaction) { + if (!mostRecentTransaction) { // There is a more recent PUT transaction than the one we are currently processing. // When a PUT is issued, it replaces any layers that would have been there before. // Therefore any data relating to this older transaction is no longer needed. LOGGER.info(String.format("Newer PUT found for %s %s since transaction %s. " + "Deleting all files associated with the earlier transaction.", arbitraryTransactionData.getService(), - arbitraryTransactionData.getName(), Base58.encode(signature))); + arbitraryTransactionData.getName(), Base58.encode(arbitraryTransactionData.getSignature()))); ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData); - EventBus.INSTANCE.notify( - new DataMonitorEvent( - System.currentTimeMillis(), - arbitraryTransactionData.getIdentifier(), - arbitraryTransactionData.getName(), - arbitraryTransactionData.getService().name(), - "deleting data due to replacement", - arbitraryTransactionData.getTimestamp(), - moreRecentPutTransaction.get().getTimestamp() - ) - ); + + Optional moreRecentPutTransaction + = processedTransactions.stream() + .filter(data -> data.equals(arbitraryTransactionData)) + .findAny(); + + if( moreRecentPutTransaction.isPresent() ) { + EventBus.INSTANCE.notify( + new DataMonitorEvent( + System.currentTimeMillis(), + arbitraryTransactionData.getIdentifier(), + arbitraryTransactionData.getName(), + arbitraryTransactionData.getService().name(), + "deleting data due to replacement", + arbitraryTransactionData.getTimestamp(), + moreRecentPutTransaction.get().getTimestamp() + ) + ); + } + else { + LOGGER.warn("Something went wrong with the most recent put transaction determination!"); + } + continue; } @@ -226,18 +255,21 @@ public class ArbitraryDataCleanupManager extends Thread { LOGGER.debug(String.format("Transaction %s has complete file and all chunks", Base58.encode(arbitraryTransactionData.getSignature()))); - ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT); - EventBus.INSTANCE.notify( - new DataMonitorEvent( - System.currentTimeMillis(), - arbitraryTransactionData.getIdentifier(), - arbitraryTransactionData.getName(), - arbitraryTransactionData.getService().name(), - "deleting data due to age", - arbitraryTransactionData.getTimestamp(), - arbitraryTransactionData.getTimestamp() - ) - ); + boolean wasDeleted = ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT); + + if( wasDeleted ) { + EventBus.INSTANCE.notify( + new DataMonitorEvent( + System.currentTimeMillis(), + arbitraryTransactionData.getIdentifier(), + arbitraryTransactionData.getName(), + arbitraryTransactionData.getService().name(), + "deleting file, retaining chunks", + arbitraryTransactionData.getTimestamp(), + arbitraryTransactionData.getTimestamp() + ) + ); + } continue; } @@ -452,18 +484,6 @@ public class ArbitraryDataCleanupManager extends Thread { // Relates to a different name - don't delete it return false; } - - EventBus.INSTANCE.notify( - new DataMonitorEvent( - System.currentTimeMillis(), - arbitraryTransactionData.getIdentifier(), - arbitraryTransactionData.getName(), - arbitraryTransactionData.getService().name(), - "randomly selected for deletion", - arbitraryTransactionData.getTimestamp(), - arbitraryTransactionData.getTimestamp() - ) - ); } } catch (DataException e) { @@ -473,6 +493,7 @@ public class ArbitraryDataCleanupManager extends Thread { } LOGGER.info("Deleting random file {} because we have reached max storage capacity...", randomItem.toString()); + fireRandomItemDeletionNotification(randomItem, repository, "Deleting random file, because we have reached max storage capacity"); boolean success = randomItem.delete(); if (success) { try { @@ -487,6 +508,35 @@ public class ArbitraryDataCleanupManager extends Thread { return false; } + private void fireRandomItemDeletionNotification(File randomItem, Repository repository, String reason) { + try { + Path parentFileNamePath = randomItem.toPath().toAbsolutePath().getParent().getFileName(); + if (parentFileNamePath != null) { + String signature58 = parentFileNamePath.toString(); + byte[] signature = Base58.decode(signature58); + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (transactionData != null && transactionData.getType() == Transaction.TransactionType.ARBITRARY) { + ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; + + EventBus.INSTANCE.notify( + new DataMonitorEvent( + System.currentTimeMillis(), + arbitraryTransactionData.getIdentifier(), + arbitraryTransactionData.getName(), + arbitraryTransactionData.getService().name(), + reason, + arbitraryTransactionData.getTimestamp(), + arbitraryTransactionData.getTimestamp() + ) + ); + } + } + + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + private void cleanupTempDirectory(String folder, long now, long minAge) { String baseDir = Settings.getInstance().getTempDataPath(); Path tempDir = Paths.get(baseDir, folder); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataExamination.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataExamination.java new file mode 100644 index 00000000..f42feb73 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataExamination.java @@ -0,0 +1,21 @@ +package org.qortal.controller.arbitrary; + +public class ArbitraryDataExamination { + + private boolean pass; + + private String notes; + + public ArbitraryDataExamination(boolean pass, String notes) { + this.pass = pass; + this.notes = notes; + } + + public boolean isPass() { + return pass; + } + + public String getNotes() { + return notes; + } +} diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 47a25a03..85df9db5 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -198,13 +198,35 @@ public class ArbitraryDataManager extends Thread { final int limit = 100; int offset = 0; + List allArbitraryTransactionsInDescendingOrder; + + try (final Repository repository = RepositoryManager.getRepository()) { + + if( name == null ) { + allArbitraryTransactionsInDescendingOrder + = repository.getArbitraryRepository() + .getLatestArbitraryTransactions(); + } + else { + allArbitraryTransactionsInDescendingOrder + = repository.getArbitraryRepository() + .getLatestArbitraryTransactionsByName(name); + } + } catch( Exception e) { + LOGGER.error(e.getMessage(), e); + allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0); + } + + // collect processed transactions in a set to ensure outdated data transactions do not get fetched + Set processedTransactions = new HashSet<>(); + while (!isStopping) { Thread.sleep(1000L); // Any arbitrary transactions we want to fetch data for? try (final Repository repository = RepositoryManager.getRepository()) { - List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, name, null, ConfirmationStatus.BOTH, limit, offset, true); - // LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit); + List signatures = processTransactionsForSignatures(limit, offset, allArbitraryTransactionsInDescendingOrder, processedTransactions); + if (signatures == null || signatures.isEmpty()) { offset = 0; break; @@ -226,7 +248,8 @@ public class ArbitraryDataManager extends Thread { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData(); // Skip transactions that we don't need to proactively store data for - if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) { + ArbitraryDataExamination arbitraryDataExamination = storageManager.shouldPreFetchData(repository, arbitraryTransactionData); + if (!arbitraryDataExamination.isPass()) { iterator.remove(); EventBus.INSTANCE.notify( @@ -235,7 +258,7 @@ public class ArbitraryDataManager extends Thread { arbitraryTransactionData.getIdentifier(), arbitraryTransactionData.getName(), arbitraryTransactionData.getService().name(), - "don't need to proactively store, skipping", + arbitraryDataExamination.getNotes(), arbitraryTransactionData.getTimestamp(), arbitraryTransactionData.getTimestamp() ) @@ -342,7 +365,7 @@ public class ArbitraryDataManager extends Thread { try (final Repository repository = RepositoryManager.getRepository()) { allArbitraryTransactionsInDescendingOrder = repository.getArbitraryRepository() - .getLatestArbitraryTransactions(Settings.getInstance().getDataFetchLimit()); + .getLatestArbitraryTransactions(); } catch( Exception e) { LOGGER.error(e.getMessage(), e); allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0); @@ -410,18 +433,6 @@ public class ArbitraryDataManager extends Thread { // fetch the metadata and notify the event bus again ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); - EventBus.INSTANCE.notify( - new DataMonitorEvent( - System.currentTimeMillis(), - arbitraryTransactionData.getIdentifier(), - arbitraryTransactionData.getName(), - arbitraryTransactionData.getService().name(), - "fetching metadata", - arbitraryTransactionData.getTimestamp(), - arbitraryTransactionData.getTimestamp() - ) - ); - // Ask our connected peers if they have metadata for this signature fetchMetadata(arbitraryTransactionData); @@ -444,10 +455,14 @@ public class ArbitraryDataManager extends Thread { } } - private static List processTransactionsForSignatures(int limit, int offset, List allArbitraryTransactionsInDescendingOrder, Set processedTransactions) { + private static List processTransactionsForSignatures( + int limit, + int offset, + List transactionsInDescendingOrder, + Set processedTransactions) { // these transactions are in descending order, latest transactions come first List transactions - = allArbitraryTransactionsInDescendingOrder.stream() + = transactionsInDescendingOrder.stream() .skip(offset) .limit(limit) .collect(Collectors.toList()); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index 62feb8bd..d3747cb3 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -155,31 +155,31 @@ public class ArbitraryDataStorageManager extends Thread { * @param arbitraryTransactionData - the transaction * @return boolean - whether to prefetch or not */ - public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { + public ArbitraryDataExamination shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { String name = arbitraryTransactionData.getName(); // Only fetch data associated with hashes, as we already have RAW_DATA if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) { - return false; + return new ArbitraryDataExamination(false, "Only fetch data associated with hashes"); } // Don't fetch anything more if we're (nearly) out of space // Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to // avoid a fetch/delete loop if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) { - return false; + return new ArbitraryDataExamination(false,"Don't fetch anything more if we're (nearly) out of space"); } // Don't fetch anything if we're (nearly) out of space for this name // Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to // avoid a fetch/delete loop if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) { - return false; + return new ArbitraryDataExamination(false, "Don't fetch anything if we're (nearly) out of space for this name"); } // Don't store data unless it's an allowed type (public/private) if (!this.isDataTypeAllowed(arbitraryTransactionData)) { - return false; + return new ArbitraryDataExamination(false, "Don't store data unless it's an allowed type (public/private)"); } // Handle transactions without names differently @@ -189,21 +189,21 @@ public class ArbitraryDataStorageManager extends Thread { // Never fetch data from blocked names, even if they are followed if (ListUtils.isNameBlocked(name)) { - return false; + return new ArbitraryDataExamination(false, "blocked name"); } switch (Settings.getInstance().getStoragePolicy()) { case FOLLOWED: case FOLLOWED_OR_VIEWED: - return ListUtils.isFollowingName(name); + return new ArbitraryDataExamination(ListUtils.isFollowingName(name), Settings.getInstance().getStoragePolicy().name()); case ALL: - return true; + return new ArbitraryDataExamination(true, Settings.getInstance().getStoragePolicy().name()); case NONE: case VIEWED: default: - return false; + return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name()); } } @@ -214,17 +214,17 @@ public class ArbitraryDataStorageManager extends Thread { * * @return boolean - whether the storage policy allows for unnamed data */ - private boolean shouldPreFetchDataWithoutName() { + private ArbitraryDataExamination shouldPreFetchDataWithoutName() { switch (Settings.getInstance().getStoragePolicy()) { case ALL: - return true; + return new ArbitraryDataExamination(true, "Fetching all data"); case NONE: case VIEWED: case FOLLOWED: case FOLLOWED_OR_VIEWED: default: - return false; + return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name()); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryTransactionDataHashWrapper.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryTransactionDataHashWrapper.java index 0f64652c..9ff40771 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryTransactionDataHashWrapper.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryTransactionDataHashWrapper.java @@ -7,7 +7,7 @@ import java.util.Objects; public class ArbitraryTransactionDataHashWrapper { - private final ArbitraryTransactionData data; + private ArbitraryTransactionData data; private int service; @@ -23,6 +23,12 @@ public class ArbitraryTransactionDataHashWrapper { this.identifier = data.getIdentifier(); } + public ArbitraryTransactionDataHashWrapper(int service, String name, String identifier) { + this.service = service; + this.name = name; + this.identifier = identifier; + } + public ArbitraryTransactionData getData() { return data; } diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 00572337..53bbfad5 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -27,7 +27,9 @@ public interface ArbitraryRepository { public List getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException; - List getLatestArbitraryTransactions(int limit) throws DataException; + List getLatestArbitraryTransactions() throws DataException; + + List getLatestArbitraryTransactionsByName(String name) throws DataException; public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index a42bc208..599ac8c0 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -228,18 +228,86 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } @Override - public List getLatestArbitraryTransactions(int limit) throws DataException { + public List getLatestArbitraryTransactions() throws DataException { String sql = "SELECT type, reference, signature, creator, created_when, fee, " + "tx_group_id, block_height, approval_status, approval_height, " + "version, nonce, service, size, is_data_raw, data, metadata_hash, " + "name, identifier, update_method, secret, compression FROM ArbitraryTransactions " + "JOIN Transactions USING (signature) " + "WHERE name IS NOT NULL " + - "ORDER BY created_when DESC " + - "LIMIT ?"; + "ORDER BY created_when DESC"; List arbitraryTransactionData = new ArrayList<>(); - try (ResultSet resultSet = this.repository.checkedExecute(sql, limit)) { + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return new ArrayList<>(0); + + do { + byte[] reference = resultSet.getBytes(2); + byte[] signature = resultSet.getBytes(3); + byte[] creatorPublicKey = resultSet.getBytes(4); + long timestamp = resultSet.getLong(5); + + Long fee = resultSet.getLong(6); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(7); + + Integer blockHeight = resultSet.getInt(8); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9)); + Integer approvalHeight = resultSet.getInt(10); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + int version = resultSet.getInt(11); + int nonce = resultSet.getInt(12); + int serviceInt = resultSet.getInt(13); + int size = resultSet.getInt(14); + boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false + DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; + byte[] data = resultSet.getBytes(16); + byte[] metadataHash = resultSet.getBytes(17); + String nameResult = resultSet.getString(18); + String identifierResult = resultSet.getString(19); + Method method = Method.valueOf(resultSet.getInt(20)); + byte[] secret = resultSet.getBytes(21); + Compression compression = Compression.valueOf(resultSet.getInt(22)); + // FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls. + + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + version, serviceInt, nonce, size, nameResult, identifierResult, method, secret, + compression, data, dataType, metadataHash, null); + + arbitraryTransactionData.add(transactionData); + } while (resultSet.next()); + + return arbitraryTransactionData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + return new ArrayList<>(0); + } + } + + @Override + public List getLatestArbitraryTransactionsByName( String name ) throws DataException { + String sql = "SELECT type, reference, signature, creator, created_when, fee, " + + "tx_group_id, block_height, approval_status, approval_height, " + + "version, nonce, service, size, is_data_raw, data, metadata_hash, " + + "name, identifier, update_method, secret, compression FROM ArbitraryTransactions " + + "JOIN Transactions USING (signature) " + + "WHERE name = ? " + + "ORDER BY created_when DESC"; + List arbitraryTransactionData = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql, name)) { if (resultSet == null) return new ArrayList<>(0); diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index eede9756..78e1f73e 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -508,9 +508,9 @@ public class Settings { */ private boolean connectionPoolMonitorEnabled = false; - private int dataFetchLimit = 1_000_000; + private int buildArbitraryResourcesBatchSize = 200; - // Domain mapping + // Domain mapping public static class ThreadLimit { private String messageType; private Integer limit; @@ -1336,7 +1336,7 @@ public class Settings { return connectionPoolMonitorEnabled; } - public int getDataFetchLimit() { - return dataFetchLimit; + public int getBuildArbitraryResourcesBatchSize() { + return buildArbitraryResourcesBatchSize; } } diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index ee9b0b8e..60285bcf 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -9,6 +9,7 @@ import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Service; import org.qortal.block.BlockChain; import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.controller.arbitrary.ArbitraryTransactionDataHashWrapper; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.crypto.Crypto; import org.qortal.crypto.MemoryPoW; @@ -31,8 +32,12 @@ import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.NTP; import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; public class ArbitraryTransaction extends Transaction { @@ -303,8 +308,7 @@ public class ArbitraryTransaction extends Transaction { // Add/update arbitrary resource caches, but don't update the status as this involves time-consuming // disk reads, and is more prone to failure. The status will be updated on metadata retrieval, or when // accessing the resource. - this.updateArbitraryResourceCache(repository); - this.updateArbitraryMetadataCache(repository); + this.updateArbitraryResourceCacheIncludingMetadata(repository, new HashSet<>(0), new HashMap<>(0)); repository.saveChanges(); @@ -360,7 +364,10 @@ public class ArbitraryTransaction extends Transaction { * * @throws DataException */ - public void updateArbitraryResourceCache(Repository repository) throws DataException { + public void updateArbitraryResourceCacheIncludingMetadata( + Repository repository, + Set latestTransactionWrappers, + Map resourceByWrapper) throws DataException { // Don't cache resources without a name (such as auto updates) if (arbitraryTransactionData.getName() == null) { return; @@ -385,29 +392,42 @@ public class ArbitraryTransaction extends Transaction { arbitraryResourceData.name = name; arbitraryResourceData.identifier = identifier; - // Get the latest transaction - ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); - if (latestTransactionData == null) { - LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData); - // We don't have a latest transaction, so delete from cache - repository.getArbitraryRepository().delete(arbitraryResourceData); - return; + final ArbitraryTransactionDataHashWrapper wrapper = new ArbitraryTransactionDataHashWrapper(arbitraryTransactionData); + + ArbitraryTransactionData latestTransactionData; + if( latestTransactionWrappers.contains(wrapper)) { + latestTransactionData + = latestTransactionWrappers.stream() + .filter( latestWrapper -> latestWrapper.equals(wrapper)) + .findAny().get() + .getData(); } + else { + // Get the latest transaction + latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); + if (latestTransactionData == null) { + LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData); + // We don't have a latest transaction, so delete from cache + repository.getArbitraryRepository().delete(arbitraryResourceData); + return; + } + } + ArbitraryResourceData existingArbitraryResourceData = resourceByWrapper.get(wrapper); - // Get existing cached entry if it exists - ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository() - .getArbitraryResource(service, name, identifier); - - LOGGER.info("updating existing arbitraryResourceData" + existingArbitraryResourceData); + if( existingArbitraryResourceData == null ) { + // Get existing cached entry if it exists + existingArbitraryResourceData = repository.getArbitraryRepository() + .getArbitraryResource(service, name, identifier); + } // Check for existing cached data if (existingArbitraryResourceData == null) { // Nothing exists yet, so set creation date from the current transaction (it will be reduced later if needed) arbitraryResourceData.created = arbitraryTransactionData.getTimestamp(); arbitraryResourceData.updated = null; - LOGGER.info("updated = null, reason = existingArbitraryResourceData == null" ); } else { + resourceByWrapper.put(wrapper, existingArbitraryResourceData); // An entry already exists - update created time from current transaction if this is older arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp()); @@ -415,22 +435,44 @@ public class ArbitraryTransaction extends Transaction { if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) { // Latest transaction matches created time, so it hasn't been updated arbitraryResourceData.updated = null; - LOGGER.info( - "updated = null, reason: existingArbitraryResourceData.created == latestTransactionData.getTimestamp() == " + - existingArbitraryResourceData.created ); } else { arbitraryResourceData.updated = latestTransactionData.getTimestamp(); - LOGGER.info("setting updated to a non-null value"); } } arbitraryResourceData.size = latestTransactionData.getSize(); - LOGGER.info("saving updated arbitraryResourceData: updated = " + arbitraryResourceData.updated); - // Save repository.getArbitraryRepository().save(arbitraryResourceData); + + // Update metadata for latest transaction if it is local + if (latestTransactionData.getMetadataHash() != null) { + ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature()); + if (metadataFile.exists()) { + ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath()); + try { + transactionMetadata.read(); + + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setArbitraryResourceData(arbitraryResourceData); + metadata.setTitle(transactionMetadata.getTitle()); + metadata.setDescription(transactionMetadata.getDescription()); + metadata.setCategory(transactionMetadata.getCategory()); + metadata.setTags(transactionMetadata.getTags()); + repository.getArbitraryRepository().save(metadata); + + } catch (IOException e) { + // Ignore, as we can add it again later + } + } else { + // We don't have a local copy of this metadata file, so delete it from the cache + // It will be re-added if the file later arrives via the network + ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); + metadata.setArbitraryResourceData(arbitraryResourceData); + repository.getArbitraryRepository().delete(metadata); + } + } } public void updateArbitraryResourceStatus(Repository repository) throws DataException { @@ -465,60 +507,4 @@ public class ArbitraryTransaction extends Transaction { repository.getArbitraryRepository().setStatus(arbitraryResourceData, status); } - public void updateArbitraryMetadataCache(Repository repository) throws DataException { - // Get the latest transaction - ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier()); - if (latestTransactionData == null) { - // We don't have a latest transaction, so give up - return; - } - - Service service = latestTransactionData.getService(); - String name = latestTransactionData.getName(); - String identifier = latestTransactionData.getIdentifier(); - - if (service == null) { - // Unsupported service - ignore this resource - return; - } - - // In the cache we store null identifiers as "default", as it is part of the primary key - if (identifier == null) { - identifier = "default"; - } - - ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(); - arbitraryResourceData.service = service; - arbitraryResourceData.name = name; - arbitraryResourceData.identifier = identifier; - - // Update metadata for latest transaction if it is local - if (latestTransactionData.getMetadataHash() != null) { - ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature()); - if (metadataFile.exists()) { - ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath()); - try { - transactionMetadata.read(); - - ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); - metadata.setArbitraryResourceData(arbitraryResourceData); - metadata.setTitle(transactionMetadata.getTitle()); - metadata.setDescription(transactionMetadata.getDescription()); - metadata.setCategory(transactionMetadata.getCategory()); - metadata.setTags(transactionMetadata.getTags()); - repository.getArbitraryRepository().save(metadata); - - } catch (IOException e) { - // Ignore, as we can add it again later - } - } else { - // We don't have a local copy of this metadata file, so delete it from the cache - // It will be re-added if the file later arrives via the network - ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata(); - metadata.setArbitraryResourceData(arbitraryResourceData); - repository.getArbitraryRepository().delete(metadata); - } - } - } - } diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index 3837e1cf..c860a034 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -209,7 +209,15 @@ public class ArbitraryTransactionUtils { return ArbitraryTransactionUtils.isFileRecent(filePath, now, cleanupAfter); } - public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException { + /** + * + * @param arbitraryTransactionData + * @param now + * @param cleanupAfter + * @return true if file is deleted, otherwise return false + * @throws DataException + */ + public static boolean deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException { byte[] completeHash = arbitraryTransactionData.getData(); byte[] signature = arbitraryTransactionData.getSignature(); @@ -220,6 +228,11 @@ public class ArbitraryTransactionUtils { "if needed", Base58.encode(completeHash)); arbitraryDataFile.delete(); + + return true; + } + else { + return false; } } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java index 1d8f23b3..1968eb60 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java @@ -73,14 +73,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.FOLLOWED_OR_VIEWED, Settings.getInstance().getStoragePolicy()); assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We should store but not pre-fetch data for this transaction assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); } } @@ -108,14 +108,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy()); assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We shouldn't store or pre-fetch data for this transaction assertFalse(storageManager.canStoreData(arbitraryTransactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); } } @@ -143,14 +143,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store but not pre-fetch data for this transaction assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy()); assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We should store but not pre-fetch data for this transaction assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); } } @@ -178,14 +178,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy()); assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We should store and pre-fetch data for this transaction assertTrue(storageManager.canStoreData(arbitraryTransactionData)); - assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); } } @@ -213,14 +213,14 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We shouldn't store or pre-fetch data for this transaction assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy()); assertFalse(storageManager.canStoreData(arbitraryTransactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We shouldn't store or pre-fetch data for this transaction assertFalse(storageManager.canStoreData(arbitraryTransactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass()); } } @@ -236,7 +236,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { // We should store but not pre-fetch data for this transaction assertTrue(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, transactionData).isPass()); } }