diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 6c3a32e6..65ded8c9 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -102,6 +102,9 @@ public class ArbitraryDataManager extends Thread { continue; } + // Fetch metadata + this.fetchAllMetadata(); + // Fetch data according to storage policy switch (Settings.getInstance().getStoragePolicy()) { case FOLLOWED: @@ -225,6 +228,83 @@ public class ArbitraryDataManager extends Thread { } } + private void fetchAllMetadata() { + ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); + + // Paginate queries when fetching arbitrary transactions + final int limit = 100; + int offset = 0; + + while (!isStopping) { + + // Any arbitrary transactions we want to fetch data for? + try (final Repository repository = RepositoryManager.getRepository()) { + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true); + // LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit); + if (signatures == null || signatures.isEmpty()) { + offset = 0; + break; + } + offset += limit; + + // Loop through signatures and remove ones we don't need to process + Iterator iterator = signatures.iterator(); + while (iterator.hasNext()) { + byte[] signature = (byte[]) iterator.next(); + + ArbitraryTransaction arbitraryTransaction = fetchTransaction(repository, signature); + if (arbitraryTransaction == null) { + // Best not to process this one + iterator.remove(); + continue; + } + ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData(); + + // Skip transactions that are blocked + if (storageManager.isBlocked(arbitraryTransactionData)) { + iterator.remove(); + continue; + } + + // Remove transactions that we already have local data for + if (hasLocalMetadata(arbitraryTransaction)) { + iterator.remove(); + continue; + } + } + + if (signatures.isEmpty()) { + continue; + } + + // Pick one at random + final int index = new Random().nextInt(signatures.size()); + byte[] signature = signatures.get(index); + + if (signature == null) { + continue; + } + + // Check to see if we have had a more recent PUT + ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); + boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData); + if (hasMoreRecentPutTransaction) { + // There is a more recent PUT transaction than the one we are currently processing. + // When a PUT is issued, it replaces any layers that would have been there before. + // Therefore any data relating to this older transaction is no longer needed and we + // shouldn't fetch it from the network. + continue; + } + + // Ask our connected peers if they have metadata for this signature + fetchMetadata(arbitraryTransactionData); + + } catch (DataException e) { + LOGGER.error("Repository issue when fetching arbitrary transaction data", e); + } + } + } + private ArbitraryTransaction fetchTransaction(final Repository repository, byte[] signature) { try { TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); @@ -244,16 +324,42 @@ public class ArbitraryDataManager extends Thread { } catch (DataException e) { LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e); - return true; + return true; // Assume true for now, to avoid network spam on error } } + private boolean hasLocalMetadata(ArbitraryTransaction arbitraryTransaction) { + try { + ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData(); + byte[] signature = arbitraryTransactionData.getSignature(); + byte[] metadataHash = arbitraryTransactionData.getMetadataHash(); + ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature); + + return metadataFile.exists(); + + } catch (DataException e) { + LOGGER.error("Repository issue when checking arbitrary transaction's metadata is local", e); + return true; // Assume true for now, to avoid network spam on error + } + } // Entrypoint to request new data from peers public boolean fetchData(ArbitraryTransactionData arbitraryTransactionData) { return ArbitraryDataFileListManager.getInstance().fetchArbitraryDataFileList(arbitraryTransactionData); } + // Entrypoint to request new metadata from peers + public byte[] fetchMetadata(ArbitraryTransactionData arbitraryTransactionData) { + + ArbitraryDataResource resource = new ArbitraryDataResource( + arbitraryTransactionData.getName(), + ArbitraryDataFile.ResourceIdType.NAME, + arbitraryTransactionData.getService(), + arbitraryTransactionData.getIdentifier() + ); + return ArbitraryMetadataManager.getInstance().fetchMetadata(resource, true); + } + // Useful methods used by other parts of the app diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index 2c992c61..4568d3fd 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -229,6 +229,16 @@ public class ArbitraryDataStorageManager extends Thread { } } + /** + * Check if data relating to a transaction is blocked by this node. + * + * @param arbitraryTransactionData - the transaction + * @return boolean - whether the resource is blocked or not + */ + public boolean isBlocked(ArbitraryTransactionData arbitraryTransactionData) { + return isNameBlocked(arbitraryTransactionData.getName()); + } + private boolean isDataTypeAllowed(ArbitraryTransactionData arbitraryTransactionData) { byte[] secret = arbitraryTransactionData.getSecret(); boolean hasSecret = (secret != null && secret.length == 32);