From 8e0e455d411b1def0f25c4339e878b119aa720eb Mon Sep 17 00:00:00 2001 From: kennycud Date: Fri, 6 Jun 2025 19:01:09 -0700 Subject: [PATCH] blocks minted adjustments removal is a new feature trigger primary names are now used throughout the chat repository numerous message handlers have been optimized, many message handlers are now getting added to a list and scheduled for processing and when they get processed, the database gets queried significantly less, because the message requests and responses are getting batched together for database access rather than querying the database one by one, the thread limits for these message types have been significantly increased, because each individual thread coming in does very little, all it does is add the message to a list to be scheduled at a later time --- .../api/resource/ArbitraryResource.java | 4 +- .../qortal/api/resource/CrossChainUtils.java | 48 ++ src/main/java/org/qortal/block/Block.java | 20 +- .../java/org/qortal/block/BlockChain.java | 7 +- .../controller/TransactionImporter.java | 217 +++++-- .../ArbitraryDataFileListManager.java | 582 +++++++++++------- .../arbitrary/ArbitraryDataFileManager.java | 58 +- .../ArbitraryDataFileRequestThread.java | 207 ++++--- .../arbitrary/ArbitraryMetadataManager.java | 247 +++++--- .../controller/arbitrary/PeerMessage.java | 22 + .../qortal/controller/tradebot/TradeBot.java | 287 ++++++--- src/main/java/org/qortal/crosschain/ACCT.java | 7 + .../org/qortal/crosschain/BitcoinACCTv1.java | 25 +- .../org/qortal/crosschain/BitcoinACCTv3.java | 25 +- .../org/qortal/crosschain/DigibyteACCTv3.java | 25 +- .../org/qortal/crosschain/DogecoinACCTv1.java | 25 +- .../org/qortal/crosschain/DogecoinACCTv3.java | 25 +- .../org/qortal/crosschain/LitecoinACCTv1.java | 25 +- .../org/qortal/crosschain/LitecoinACCTv3.java | 25 +- .../org/qortal/crosschain/PirateChain.java | 7 +- .../qortal/crosschain/PirateChainACCTv3.java | 25 +- .../org/qortal/crosschain/PirateWallet.java | 5 +- .../qortal/crosschain/RavencoinACCTv3.java | 25 +- src/main/java/org/qortal/network/Network.java | 2 +- .../org/qortal/repository/ATRepository.java | 4 + .../qortal/repository/AccountRepository.java | 2 + .../repository/TransactionRepository.java | 3 + .../repository/hsqldb/HSQLDBATRepository.java | 111 ++++ .../hsqldb/HSQLDBAccountRepository.java | 33 + .../hsqldb/HSQLDBChatRepository.java | 52 +- .../HSQLDBTransactionRepository.java | 52 ++ .../java/org/qortal/settings/Settings.java | 12 +- .../org/qortal/utils/ArbitraryIndexUtils.java | 9 - .../utils/ArbitraryTransactionUtils.java | 18 + src/main/resources/blockchain.json | 3 +- src/test/resources/test-chain-v2.json | 3 +- 36 files changed, 1598 insertions(+), 649 deletions(-) create mode 100644 src/main/java/org/qortal/controller/arbitrary/PeerMessage.java diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 00c4be0d..79fb8528 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -2093,12 +2093,12 @@ public String finalizeUpload( } } catch (IOException | ApiException | DataException e) { - LOGGER.warn(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e); + LOGGER.warn(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); if (!response.isCommitted()) { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); } } catch (NumberFormatException e) { - LOGGER.warn(String.format("Invalid range for %s %s: %s", service, name, e.getMessage()), e); + LOGGER.warn(String.format("Invalid range for %s %s: %s", service, name, e.getMessage())); if (!response.isCommitted()) { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage()); } diff --git a/src/main/java/org/qortal/api/resource/CrossChainUtils.java b/src/main/java/org/qortal/api/resource/CrossChainUtils.java index c923850f..eec784e7 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainUtils.java +++ b/src/main/java/org/qortal/api/resource/CrossChainUtils.java @@ -12,7 +12,9 @@ import org.bouncycastle.util.Strings; import org.json.simple.JSONObject; import org.qortal.api.model.CrossChainTradeLedgerEntry; import org.qortal.api.model.crosschain.BitcoinyTBDRequest; +import org.qortal.asset.Asset; import org.qortal.crosschain.*; +import org.qortal.data.account.AccountBalanceData; import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.data.crosschain.*; @@ -30,6 +32,7 @@ import java.io.Writer; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.*; +import java.util.function.Function; import java.util.stream.Collectors; @@ -234,6 +237,9 @@ public class CrossChainUtils { return bitcoiny.getBlockchainProvider().removeServer(server); } + public static ChainableServer getCurrentServer( Bitcoiny bitcoiny ) { + return bitcoiny.getBlockchainProvider().getCurrentServer(); + } /** * Set Current Server * @@ -773,4 +779,46 @@ public class CrossChainUtils { entries.add(ledgerEntry); } } + + public static List populateTradeDataList(Repository repository, ACCT acct, List atDataList) throws DataException { + + if(atDataList.isEmpty()) return new ArrayList<>(0); + + List latestATStates + = repository.getATRepository() + .getLatestATStates( + atDataList.stream() + .map(ATData::getATAddress) + .collect(Collectors.toList()) + ); + + Map atStateDataByAtAddress + = latestATStates.stream().collect(Collectors.toMap(ATStateData::getATAddress, Function.identity())); + + Map atDataByAtAddress + = atDataList.stream().collect(Collectors.toMap(ATData::getATAddress, Function.identity())); + + Map balanceByAtAddress + = repository + .getAccountRepository() + .getBalances(new ArrayList<>(atDataByAtAddress.keySet()), Asset.QORT) + .stream().collect(Collectors.toMap(AccountBalanceData::getAddress, AccountBalanceData::getBalance)); + + List crossChainTradeDataList = new ArrayList<>(latestATStates.size()); + + for( ATStateData atStateData : latestATStates ) { + ATData atData = atDataByAtAddress.get(atStateData.getATAddress()); + crossChainTradeDataList.add( + acct.populateTradeData( + repository, + atData.getCreatorPublicKey(), + atData.getCreation(), + atStateData, + OptionalLong.of(balanceByAtAddress.get(atStateData.getATAddress())) + ) + ); + } + + return crossChainTradeDataList; + } } \ No newline at end of file diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 753b5dfa..f2291910 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1723,7 +1723,15 @@ public class Block { accountData.setBlocksMinted(accountData.getBlocksMinted() + 1); LOGGER.trace(() -> String.format("Block minter %s up to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""))); - final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment() + accountData.getBlocksMintedPenalty(); + int blocksMintedAdjustment + = + (this.blockData.getHeight() > BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight()) + ? + 0 + : + accountData.getBlocksMintedAdjustment(); + + final int effectiveBlocksMinted = accountData.getBlocksMinted() + blocksMintedAdjustment + accountData.getBlocksMintedPenalty(); for (int newLevel = maximumLevel; newLevel >= 0; --newLevel) if (effectiveBlocksMinted >= cumulativeBlocksByLevel.get(newLevel)) { @@ -2131,7 +2139,15 @@ public class Block { accountData.setBlocksMinted(accountData.getBlocksMinted() - 1); LOGGER.trace(() -> String.format("Block minter %s down to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""))); - final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment() + accountData.getBlocksMintedPenalty(); + int blocksMintedAdjustment + = + (this.blockData.getHeight() > BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight()) + ? + 0 + : + accountData.getBlocksMintedAdjustment(); + + final int effectiveBlocksMinted = accountData.getBlocksMinted() + blocksMintedAdjustment + accountData.getBlocksMintedPenalty(); for (int newLevel = maximumLevel; newLevel >= 0; --newLevel) if (effectiveBlocksMinted >= cumulativeBlocksByLevel.get(newLevel)) { diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index d7e405ed..1349383d 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -93,7 +93,8 @@ public class BlockChain { nullGroupMembershipHeight, ignoreLevelForRewardShareHeight, adminQueryFixHeight, - multipleNamesPerAccountHeight + multipleNamesPerAccountHeight, + mintedBlocksAdjustmentRemovalHeight } // Custom transaction fees @@ -695,6 +696,10 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.multipleNamesPerAccountHeight.name()).intValue(); } + public int getMintedBlocksAdjustmentRemovalHeight() { + return this.featureTriggers.get(FeatureTrigger.mintedBlocksAdjustmentRemovalHeight.name()).intValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { diff --git a/src/main/java/org/qortal/controller/TransactionImporter.java b/src/main/java/org/qortal/controller/TransactionImporter.java index 6c846f3b..c1ace5fb 100644 --- a/src/main/java/org/qortal/controller/TransactionImporter.java +++ b/src/main/java/org/qortal/controller/TransactionImporter.java @@ -2,6 +2,7 @@ package org.qortal.controller; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.arbitrary.PeerMessage; import org.qortal.data.block.BlockData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; @@ -20,7 +21,11 @@ import org.qortal.utils.Base58; import org.qortal.utils.NTP; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; import java.util.stream.Collectors; public class TransactionImporter extends Thread { @@ -50,6 +55,10 @@ public class TransactionImporter extends Thread { /** Cached list of unconfirmed transactions, used when counting per creator. This is replaced regularly */ public static List unconfirmedTransactionsCache = null; + public TransactionImporter() { + signatureMessageScheduler.scheduleAtFixedRate(this::processNetworkTransactionSignaturesMessage, 60, 1, TimeUnit.SECONDS); + getTransactionMessageScheduler.scheduleAtFixedRate(this::processNetworkGetTransactionMessages, 60, 1, TimeUnit.SECONDS); + } public static synchronized TransactionImporter getInstance() { if (instance == null) { @@ -371,36 +380,104 @@ public class TransactionImporter extends Thread { } } + // List to collect messages + private final List getTransactionMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object getTransactionMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService getTransactionMessageScheduler = Executors.newScheduledThreadPool(1); + public void onNetworkGetTransactionMessage(Peer peer, Message message) { - GetTransactionMessage getTransactionMessage = (GetTransactionMessage) message; - byte[] signature = getTransactionMessage.getSignature(); - try (final Repository repository = RepositoryManager.getRepository()) { + synchronized (getTransactionMessageLock) { + getTransactionMessageList.add(new PeerMessage(peer, message)); + } + } + + private void processNetworkGetTransactionMessages() { + + try { + List messagesToProcess; + synchronized (getTransactionMessageLock) { + messagesToProcess = new ArrayList<>(getTransactionMessageList); + getTransactionMessageList.clear(); + } + + if( messagesToProcess.isEmpty() ) return; + + Map peerMessageBySignature58 = new HashMap<>(messagesToProcess.size()); + + for( PeerMessage peerMessage : messagesToProcess ) { + GetTransactionMessage getTransactionMessage = (GetTransactionMessage) peerMessage.getMessage(); + byte[] signature = getTransactionMessage.getSignature(); + + peerMessageBySignature58.put(Base58.encode(signature), peerMessage); + } + // Firstly check the sig-valid transactions that are currently queued for import - TransactionData transactionData = this.getCachedSigValidTransactions().stream() - .filter(t -> Arrays.equals(signature, t.getSignature())) - .findFirst().orElse(null); + Map transactionsCachedBySignature58 + = this.getCachedSigValidTransactions().stream() + .collect(Collectors.toMap(t -> Base58.encode(t.getSignature()), Function.identity())); - if (transactionData == null) { + Map>> transactionsCachedBySignature58Partition + = peerMessageBySignature58.entrySet().stream() + .collect(Collectors.partitioningBy(entry -> transactionsCachedBySignature58.containsKey(entry.getKey()))); + + List signaturesNeeded + = transactionsCachedBySignature58Partition.get(false).stream() + .map(Map.Entry::getValue) + .map(PeerMessage::getMessage) + .map(message -> (GetTransactionMessage) message) + .map(GetTransactionMessage::getSignature) + .collect(Collectors.toList()); + + // transaction found in the import queue + Map transactionsToSendBySignature58 = new HashMap<>(messagesToProcess.size()); + for( Map.Entry entry : transactionsCachedBySignature58Partition.get(true)) { + transactionsToSendBySignature58.put(entry.getKey(), transactionsCachedBySignature58.get(entry.getKey())); + } + + if( !signaturesNeeded.isEmpty() ) { // Not found in import queue, so try the database - transactionData = repository.getTransactionRepository().fromSignature(signature); + try (final Repository repository = RepositoryManager.getRepository()) { + transactionsToSendBySignature58.putAll( + repository.getTransactionRepository().fromSignatures(signaturesNeeded).stream() + .collect(Collectors.toMap(transactionData -> Base58.encode(transactionData.getSignature()), Function.identity())) + ); + } catch (DataException e) { + LOGGER.error(e.getMessage(), e); + } } - if (transactionData == null) { - // Still not found - so we don't have this transaction - LOGGER.debug(() -> String.format("Ignoring GET_TRANSACTION request from peer %s for unknown transaction %s", peer, Base58.encode(signature))); - // Send no response at all??? - return; - } + for( final Map.Entry entry : transactionsToSendBySignature58.entrySet() ) { - Message transactionMessage = new TransactionMessage(transactionData); + PeerMessage peerMessage = peerMessageBySignature58.get(entry.getKey()); + final Message message = peerMessage.getMessage(); + final Peer peer = peerMessage.getPeer(); + + Runnable sendTransactionMessageRunner = () -> sendTransactionMessage(entry.getKey(), entry.getValue(), message, peer); + Thread sendTransactionMessageThread = new Thread(sendTransactionMessageRunner); + sendTransactionMessageThread.start(); + } + } catch (Exception e) { + LOGGER.error(e.getMessage(),e); + } + } + + private static void sendTransactionMessage(String signature58, TransactionData data, Message message, Peer peer) { + try { + Message transactionMessage = new TransactionMessage(data); transactionMessage.setId(message.getId()); + if (!peer.sendMessage(transactionMessage)) peer.disconnect("failed to send transaction"); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e); - } catch (TransformationException e) { - LOGGER.error(String.format("Serialization issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e); + } + catch (TransformationException e) { + LOGGER.error(String.format("Serialization issue while sending transaction %s to peer %s", signature58, peer), e); + } + catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } @@ -421,44 +498,86 @@ public class TransactionImporter extends Thread { } } + // List to collect messages + private final List signatureMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object signatureMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService signatureMessageScheduler = Executors.newScheduledThreadPool(1); + public void onNetworkTransactionSignaturesMessage(Peer peer, Message message) { - TransactionSignaturesMessage transactionSignaturesMessage = (TransactionSignaturesMessage) message; - List signatures = transactionSignaturesMessage.getSignatures(); + synchronized (signatureMessageLock) { + signatureMessageList.add(new PeerMessage(peer, message)); + } + } - try (final Repository repository = RepositoryManager.getRepository()) { - for (byte[] signature : signatures) { - String signature58 = Base58.encode(signature); - if (invalidUnconfirmedTransactions.containsKey(signature58)) { - // Previously invalid transaction - don't keep requesting it - // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks - continue; - } + public void processNetworkTransactionSignaturesMessage() { - // Ignore if this transaction is in the queue - if (incomingTransactionQueueContains(signature)) { - LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peer)); - continue; - } + try { + List messagesToProcess; + synchronized (signatureMessageLock) { + messagesToProcess = new ArrayList<>(signatureMessageList); + signatureMessageList.clear(); + } - // Do we have it already? (Before requesting transaction data itself) - if (repository.getTransactionRepository().exists(signature)) { - LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(signature), peer)); - continue; - } + Map signatureBySignature58 = new HashMap<>(messagesToProcess.size() * 10); + Map peerBySignature58 = new HashMap<>( messagesToProcess.size() * 10 ); - // Check isInterrupted() here and exit fast - if (Thread.currentThread().isInterrupted()) - return; + for( PeerMessage peerMessage : messagesToProcess ) { - // Fetch actual transaction data from peer - Message getTransactionMessage = new GetTransactionMessage(signature); - if (!peer.sendMessage(getTransactionMessage)) { - peer.disconnect("failed to request transaction"); - return; + TransactionSignaturesMessage transactionSignaturesMessage = (TransactionSignaturesMessage) peerMessage.getMessage(); + List signatures = transactionSignaturesMessage.getSignatures(); + + for (byte[] signature : signatures) { + String signature58 = Base58.encode(signature); + if (invalidUnconfirmedTransactions.containsKey(signature58)) { + // Previously invalid transaction - don't keep requesting it + // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks + continue; + } + + // Ignore if this transaction is in the queue + if (incomingTransactionQueueContains(signature)) { + LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peerMessage.getPeer())); + continue; + } + + signatureBySignature58.put(signature58, signature); + peerBySignature58.put(signature58, peerMessage.getPeer()); } } - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while processing unconfirmed transactions from peer %s", peer), e); + + if( !signatureBySignature58.isEmpty() ) { + try (final Repository repository = RepositoryManager.getRepository()) { + + // remove signatures in db already + repository.getTransactionRepository() + .fromSignatures(new ArrayList<>(signatureBySignature58.values())).stream() + .map(TransactionData::getSignature) + .map(signature -> Base58.encode(signature)) + .forEach(signature58 -> signatureBySignature58.remove(signature58)); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while processing unconfirmed transactions from peer"), e); + } + } + + // Check isInterrupted() here and exit fast + if (Thread.currentThread().isInterrupted()) + return; + + for (Map.Entry entry : signatureBySignature58.entrySet()) { + + Peer peer = peerBySignature58.get(entry.getKey()); + + // Fetch actual transaction data from peer + Message getTransactionMessage = new GetTransactionMessage(entry.getValue()); + if (peer != null && !peer.sendMessage(getTransactionMessage)) { + peer.disconnect("failed to request transaction"); + } + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index fd5fc50a..ee37dbec 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -25,6 +25,10 @@ import org.qortal.utils.NTP; import org.qortal.utils.Triple; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.qortal.controller.arbitrary.ArbitraryDataFileManager.MAX_FILE_HASH_RESPONSES; @@ -73,6 +77,8 @@ public class ArbitraryDataFileListManager { private ArbitraryDataFileListManager() { + getArbitraryDataFileListMessageScheduler.scheduleAtFixedRate(this::processNetworkGetArbitraryDataFileListMessage, 60, 1, TimeUnit.SECONDS); + arbitraryDataFileListMessageScheduler.scheduleAtFixedRate(this::processNetworkArbitraryDataFileListMessage, 60, 1, TimeUnit.SECONDS); } public static ArbitraryDataFileListManager getInstance() { @@ -413,70 +419,116 @@ public class ArbitraryDataFileListManager { // Network handlers + // List to collect messages + private final List arbitraryDataFileListMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object arbitraryDataFileListMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService arbitraryDataFileListMessageScheduler = Executors.newScheduledThreadPool(1); + public void onNetworkArbitraryDataFileListMessage(Peer peer, Message message) { // Don't process if QDN is disabled if (!Settings.getInstance().isQdnEnabled()) { return; } - ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; - LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size()); - - if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) { - long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime(); - LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}", - totalRequestTime, arbitraryDataFileListMessage.getRequestHops(), - arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); + synchronized (arbitraryDataFileListMessageLock) { + arbitraryDataFileListMessageList.add(new PeerMessage(peer, message)); } + } - // Do we have a pending request for this data? - Triple request = arbitraryDataFileListRequests.get(message.getId()); - if (request == null || request.getA() == null) { - return; - } - boolean isRelayRequest = (request.getB() != null); + private void processNetworkArbitraryDataFileListMessage() { - // Does this message's signature match what we're expecting? - byte[] signature = arbitraryDataFileListMessage.getSignature(); - String signature58 = Base58.encode(signature); - if (!request.getA().equals(signature58)) { - return; - } + try { + List messagesToProcess; + synchronized (arbitraryDataFileListMessageLock) { + messagesToProcess = new ArrayList<>(arbitraryDataFileListMessageList); + arbitraryDataFileListMessageList.clear(); + } - List hashes = arbitraryDataFileListMessage.getHashes(); - if (hashes == null || hashes.isEmpty()) { - return; - } + if (messagesToProcess.isEmpty()) return; - ArbitraryTransactionData arbitraryTransactionData = null; + Map peerMessageBySignature58 = new HashMap<>(messagesToProcess.size()); + Map signatureBySignature58 = new HashMap<>(messagesToProcess.size()); + Map isRelayRequestBySignature58 = new HashMap<>(messagesToProcess.size()); + Map> hashesBySignature58 = new HashMap<>(messagesToProcess.size()); + Map> requestBySignature58 = new HashMap<>(messagesToProcess.size()); - // Check transaction exists and hashes are correct - try (final Repository repository = RepositoryManager.getRepository()) { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (!(transactionData instanceof ArbitraryTransactionData)) + for (PeerMessage peerMessage : messagesToProcess) { + Peer peer = peerMessage.getPeer(); + Message message = peerMessage.getMessage(); + + ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; + LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size()); + + if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) { + long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime(); + LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}", + totalRequestTime, arbitraryDataFileListMessage.getRequestHops(), + arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); + } + + // Do we have a pending request for this data? + Triple request = arbitraryDataFileListRequests.get(message.getId()); + if (request == null || request.getA() == null) { + continue; + } + boolean isRelayRequest = (request.getB() != null); + + // Does this message's signature match what we're expecting? + byte[] signature = arbitraryDataFileListMessage.getSignature(); + String signature58 = Base58.encode(signature); + if (!request.getA().equals(signature58)) { + continue; + } + + List hashes = arbitraryDataFileListMessage.getHashes(); + if (hashes == null || hashes.isEmpty()) { + continue; + } + + peerMessageBySignature58.put(signature58, peerMessage); + signatureBySignature58.put(signature58, signature); + isRelayRequestBySignature58.put(signature58, isRelayRequest); + hashesBySignature58.put(signature58, hashes); + requestBySignature58.put(signature58, request); + } + + if (signatureBySignature58.isEmpty()) return; + + List arbitraryTransactionDataList; + + // Check transaction exists and hashes are correct + try (final Repository repository = RepositoryManager.getRepository()) { + arbitraryTransactionDataList + = repository.getTransactionRepository() + .fromSignatures(new ArrayList<>(signatureBySignature58.values())).stream() + .filter(data -> data instanceof ArbitraryTransactionData) + .map(data -> (ArbitraryTransactionData) data) + .collect(Collectors.toList()); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list"), e); return; + } - arbitraryTransactionData = (ArbitraryTransactionData) transactionData; + for (ArbitraryTransactionData arbitraryTransactionData : arbitraryTransactionDataList) { -// // Load data file(s) -// ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData); -// -// // Check all hashes exist -// for (byte[] hash : hashes) { -// //LOGGER.debug("Received hash {}", Base58.encode(hash)); -// if (!arbitraryDataFile.containsChunk(hash)) { -// // Check the hash against the complete file -// if (!Arrays.equals(arbitraryDataFile.getHash(), hash)) { -// LOGGER.info("Received non-matching chunk hash {} for signature {}. This could happen if we haven't obtained the metadata file yet.", Base58.encode(hash), signature58); -// return; -// } -// } -// } + byte[] signature = arbitraryTransactionData.getSignature(); + String signature58 = Base58.encode(signature); - if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) { - Long now = NTP.getTime(); + List hashes = hashesBySignature58.get(signature58); + + PeerMessage peerMessage = peerMessageBySignature58.get(signature58); + Peer peer = peerMessage.getPeer(); + Message message = peerMessage.getMessage(); + + ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; + + Boolean isRelayRequest = isRelayRequestBySignature58.get(signature58); + if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) { + Long now = NTP.getTime(); - if (ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.size() < MAX_FILE_HASH_RESPONSES) { // Keep track of the hashes this peer reports to have access to for (byte[] hash : hashes) { String hash58 = Base58.encode(hash); @@ -487,233 +539,303 @@ public class ArbitraryDataFileListManager { ArbitraryFileListResponseInfo responseInfo = new ArbitraryFileListResponseInfo(hash58, signature58, peer, now, arbitraryDataFileListMessage.getRequestTime(), requestHops); - ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.add(responseInfo); + ArbitraryDataFileManager.getInstance().addResponse(responseInfo); + } + + // Keep track of the source peer, for direct connections + if (arbitraryDataFileListMessage.getPeerAddress() != null) { + ArbitraryDataFileManager.getInstance().addDirectConnectionInfoIfUnique( + new ArbitraryDirectConnectionInfo(signature, arbitraryDataFileListMessage.getPeerAddress(), hashes, now)); } } - // Keep track of the source peer, for direct connections - if (arbitraryDataFileListMessage.getPeerAddress() != null) { - ArbitraryDataFileManager.getInstance().addDirectConnectionInfoIfUnique( - new ArbitraryDirectConnectionInfo(signature, arbitraryDataFileListMessage.getPeerAddress(), hashes, now)); - } - } + // Forwarding + if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) { - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e); - } + boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName())); + if (!isBlocked) { + Triple request = requestBySignature58.get(signature58); + Peer requestingPeer = request.getB(); + if (requestingPeer != null) { + Long requestTime = arbitraryDataFileListMessage.getRequestTime(); + Integer requestHops = arbitraryDataFileListMessage.getRequestHops(); - // Forwarding - if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) { - boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName())); - if (!isBlocked) { - Peer requestingPeer = request.getB(); - if (requestingPeer != null) { - Long requestTime = arbitraryDataFileListMessage.getRequestTime(); - Integer requestHops = arbitraryDataFileListMessage.getRequestHops(); + // Add each hash to our local mapping so we know who to ask later + Long now = NTP.getTime(); + for (byte[] hash : hashes) { + String hash58 = Base58.encode(hash); + ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops); + ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo); + } - // Add each hash to our local mapping so we know who to ask later - Long now = NTP.getTime(); - for (byte[] hash : hashes) { - String hash58 = Base58.encode(hash); - ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops); - ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo); - } + // Bump requestHops if it exists + if (requestHops != null) { + requestHops++; + } - // Bump requestHops if it exists - if (requestHops != null) { - requestHops++; - } + ArbitraryDataFileListMessage forwardArbitraryDataFileListMessage; - ArbitraryDataFileListMessage forwardArbitraryDataFileListMessage; + // Remove optional parameters if the requesting peer doesn't support it yet + // A message with less statistical data is better than no message at all + if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { + forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + } else { + forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, + arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); + } + forwardArbitraryDataFileListMessage.setId(message.getId()); - // Remove optional parameters if the requesting peer doesn't support it yet - // A message with less statistical data is better than no message at all - if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { - forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); - } else { - forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, - arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); - } - forwardArbitraryDataFileListMessage.setId(message.getId()); - - // Forward to requesting peer - LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer); - if (!requestingPeer.sendMessage(forwardArbitraryDataFileListMessage)) { - requestingPeer.disconnect("failed to forward arbitrary data file list"); + // Forward to requesting peer + LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer); + if (!requestingPeer.sendMessage(forwardArbitraryDataFileListMessage)) { + requestingPeer.disconnect("failed to forward arbitrary data file list"); + } + } } } } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } + // List to collect messages + private final List getArbitraryDataFileListMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object getArbitraryDataFileListMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService getArbitraryDataFileListMessageScheduler = Executors.newScheduledThreadPool(1); + public void onNetworkGetArbitraryDataFileListMessage(Peer peer, Message message) { // Don't respond if QDN is disabled if (!Settings.getInstance().isQdnEnabled()) { return; } - Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet(); - - GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; - byte[] signature = getArbitraryDataFileListMessage.getSignature(); - String signature58 = Base58.encode(signature); - Long now = NTP.getTime(); - Triple newEntry = new Triple<>(signature58, peer, now); - - // If we've seen this request recently, then ignore - if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) { - LOGGER.trace("Ignoring hash list request from peer {} for signature {}", peer, signature58); - return; + synchronized (getArbitraryDataFileListMessageLock) { + getArbitraryDataFileListMessageList.add(new PeerMessage(peer, message)); } + } - List requestedHashes = getArbitraryDataFileListMessage.getHashes(); - int hashCount = requestedHashes != null ? requestedHashes.size() : 0; - String requestingPeer = getArbitraryDataFileListMessage.getRequestingPeer(); + private void processNetworkGetArbitraryDataFileListMessage() { - if (requestingPeer != null) { - LOGGER.debug("Received hash list request with {} hashes from peer {} (requesting peer {}) for signature {}", hashCount, peer, requestingPeer, signature58); - } - else { - LOGGER.debug("Received hash list request with {} hashes from peer {} for signature {}", hashCount, peer, signature58); - } + try { + List messagesToProcess; + synchronized (getArbitraryDataFileListMessageLock) { + messagesToProcess = new ArrayList<>(getArbitraryDataFileListMessageList); + getArbitraryDataFileListMessageList.clear(); + } - List hashes = new ArrayList<>(); - ArbitraryTransactionData transactionData = null; - boolean allChunksExist = false; - boolean hasMetadata = false; + if (messagesToProcess.isEmpty()) return; - try (final Repository repository = RepositoryManager.getRepository()) { + Map signatureBySignature58 = new HashMap<>(messagesToProcess.size()); + Map> requestedHashesBySignature58 = new HashMap<>(messagesToProcess.size()); + Map requestingPeerBySignature58 = new HashMap<>(messagesToProcess.size()); + Map nowBySignature58 = new HashMap<>((messagesToProcess.size())); + Map peerMessageBySignature58 = new HashMap<>(messagesToProcess.size()); - // Firstly we need to lookup this file on chain to get a list of its hashes - transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature); - if (transactionData instanceof ArbitraryTransactionData) { + for (PeerMessage messagePeer : messagesToProcess) { + Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet(); + + Message message = messagePeer.message; + Peer peer = messagePeer.peer; + + GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; + byte[] signature = getArbitraryDataFileListMessage.getSignature(); + String signature58 = Base58.encode(signature); + Long now = NTP.getTime(); + Triple newEntry = new Triple<>(signature58, peer, now); + + // If we've seen this request recently, then ignore + if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) { + LOGGER.trace("Ignoring hash list request from peer {} for signature {}", peer, signature58); + continue; + } + + List requestedHashes = getArbitraryDataFileListMessage.getHashes(); + int hashCount = requestedHashes != null ? requestedHashes.size() : 0; + String requestingPeer = getArbitraryDataFileListMessage.getRequestingPeer(); + + if (requestingPeer != null) { + LOGGER.debug("Received hash list request with {} hashes from peer {} (requesting peer {}) for signature {}", hashCount, peer, requestingPeer, signature58); + } else { + LOGGER.debug("Received hash list request with {} hashes from peer {} for signature {}", hashCount, peer, signature58); + } + + signatureBySignature58.put(signature58, signature); + requestedHashesBySignature58.put(signature58, requestedHashes); + requestingPeerBySignature58.put(signature58, requestingPeer); + nowBySignature58.put(signature58, now); + peerMessageBySignature58.put(signature58, messagePeer); + } + + if (signatureBySignature58.isEmpty()) { + return; + } + + List hashes = new ArrayList<>(); + boolean allChunksExist = false; + boolean hasMetadata = false; + + List transactionDataList; + try (final Repository repository = RepositoryManager.getRepository()) { + + // Firstly we need to lookup this file on chain to get a list of its hashes + transactionDataList + = repository.getTransactionRepository() + .fromSignatures(new ArrayList<>(signatureBySignature58.values())).stream() + .filter(data -> data instanceof ArbitraryTransactionData) + .map(data -> (ArbitraryTransactionData) data) + .collect(Collectors.toList()); + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer"), e); + return; + } + + for (ArbitraryTransactionData transactionData : transactionDataList) { + byte[] signature = transactionData.getSignature(); + String signature58 = Base58.encode(signature); + List requestedHashes = requestedHashesBySignature58.get(signature58); // Check if we're even allowed to serve data for this transaction if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) { - // Load file(s) and add any that exist to the list of hashes - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData); + try { + // Load file(s) and add any that exist to the list of hashes + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData); - // If the peer didn't supply a hash list, we need to return all hashes for this transaction - if (requestedHashes == null || requestedHashes.isEmpty()) { - requestedHashes = new ArrayList<>(); + // If the peer didn't supply a hash list, we need to return all hashes for this transaction + if (requestedHashes == null || requestedHashes.isEmpty()) { + requestedHashes = new ArrayList<>(); - // Add the metadata file - if (arbitraryDataFile.getMetadataHash() != null) { - requestedHashes.add(arbitraryDataFile.getMetadataHash()); - hasMetadata = true; + // Add the metadata file + if (arbitraryDataFile.getMetadataHash() != null) { + requestedHashes.add(arbitraryDataFile.getMetadataHash()); + hasMetadata = true; + } + + // Add the chunk hashes + if (!arbitraryDataFile.getChunkHashes().isEmpty()) { + requestedHashes.addAll(arbitraryDataFile.getChunkHashes()); + } + // Add complete file if there are no hashes + else { + requestedHashes.add(arbitraryDataFile.getHash()); + } } - // Add the chunk hashes - if (!arbitraryDataFile.getChunkHashes().isEmpty()) { - requestedHashes.addAll(arbitraryDataFile.getChunkHashes()); - } - // Add complete file if there are no hashes - else { - requestedHashes.add(arbitraryDataFile.getHash()); + + // Assume all chunks exists, unless one can't be found below + allChunksExist = true; + + for (byte[] requestedHash : requestedHashes) { + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature); + if (chunk.exists()) { + hashes.add(chunk.getHash()); + //LOGGER.trace("Added hash {}", chunk.getHash58()); + } else { + LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58()); + allChunksExist = false; + } } + } catch (DataException e) { + LOGGER.error(e.getMessage(), e); + } + } + + // If the only file we have is the metadata then we shouldn't respond. Most nodes will already have that, + // or can use the separate metadata protocol to fetch it. This should greatly reduce network spam. + if (hasMetadata && hashes.size() == 1) { + hashes.clear(); + } + + PeerMessage peerMessage = peerMessageBySignature58.get(signature58); + Peer peer = peerMessage.getPeer(); + Message message = peerMessage.getMessage(); + + Long now = nowBySignature58.get(signature58); + + // We should only respond if we have at least one hash + String requestingPeer = requestingPeerBySignature58.get(signature58); + if (!hashes.isEmpty()) { + + // Firstly we should keep track of the requesting peer, to allow for potential direct connections later + ArbitraryDataFileManager.getInstance().addRecentDataRequest(requestingPeer); + + // We have all the chunks, so update requests map to reflect that we've sent it + // There is no need to keep track of the request, as we can serve all the chunks + if (allChunksExist) { + Triple newEntry = new Triple<>(null, null, now); + arbitraryDataFileListRequests.put(message.getId(), newEntry); } - // Assume all chunks exists, unless one can't be found below - allChunksExist = true; + String ourAddress = Network.getInstance().getOurExternalIpAddressAndPort(); + ArbitraryDataFileListMessage arbitraryDataFileListMessage; + + // Remove optional parameters if the requesting peer doesn't support it yet + // A message with less statistical data is better than no message at all + if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { + arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + } else { + arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, + hashes, NTP.getTime(), 0, ourAddress, true); + } + + arbitraryDataFileListMessage.setId(message.getId()); + + if (!peer.sendMessage(arbitraryDataFileListMessage)) { + LOGGER.debug("Couldn't send list of hashes"); + peer.disconnect("failed to send list of hashes"); + continue; + } + + if (allChunksExist) { + // Nothing left to do, so return to prevent any unnecessary forwarding from occurring + LOGGER.debug("No need for any forwarding because file list request is fully served"); + continue; + } + + } + + // We may need to forward this request on + boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName())); + if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { + // In relay mode - so ask our other peers if they have it + + + GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; + + long requestTime = getArbitraryDataFileListMessage.getRequestTime(); + int requestHops = getArbitraryDataFileListMessage.getRequestHops() + 1; + long totalRequestTime = now - requestTime; + + if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { + // Relay request hasn't timed out yet, so can potentially be rebroadcast + if (requestHops < RELAY_REQUEST_MAX_HOPS) { + // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast + + Message relayGetArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, requestingPeer); + relayGetArbitraryDataFileListMessage.setId(message.getId()); + + LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); + Network.getInstance().broadcast( + broadcastPeer -> + !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null : + broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryDataFileListMessage + ); - for (byte[] requestedHash : requestedHashes) { - ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature); - if (chunk.exists()) { - hashes.add(chunk.getHash()); - //LOGGER.trace("Added hash {}", chunk.getHash58()); } else { - LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58()); - allChunksExist = false; + // This relay request has reached the maximum number of allowed hops } + } else { + // This relay request has timed out } } } - - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e); - } - - // If the only file we have is the metadata then we shouldn't respond. Most nodes will already have that, - // or can use the separate metadata protocol to fetch it. This should greatly reduce network spam. - if (hasMetadata && hashes.size() == 1) { - hashes.clear(); - } - - // We should only respond if we have at least one hash - if (!hashes.isEmpty()) { - - // Firstly we should keep track of the requesting peer, to allow for potential direct connections later - ArbitraryDataFileManager.getInstance().addRecentDataRequest(requestingPeer); - - // We have all the chunks, so update requests map to reflect that we've sent it - // There is no need to keep track of the request, as we can serve all the chunks - if (allChunksExist) { - newEntry = new Triple<>(null, null, now); - arbitraryDataFileListRequests.put(message.getId(), newEntry); - } - - String ourAddress = Network.getInstance().getOurExternalIpAddressAndPort(); - ArbitraryDataFileListMessage arbitraryDataFileListMessage; - - // Remove optional parameters if the requesting peer doesn't support it yet - // A message with less statistical data is better than no message at all - if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { - arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); - } else { - arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, - hashes, NTP.getTime(), 0, ourAddress, true); - } - - arbitraryDataFileListMessage.setId(message.getId()); - - if (!peer.sendMessage(arbitraryDataFileListMessage)) { - LOGGER.debug("Couldn't send list of hashes"); - peer.disconnect("failed to send list of hashes"); - return; - } - LOGGER.debug("Sent list of hashes (count: {})", hashes.size()); - - if (allChunksExist) { - // Nothing left to do, so return to prevent any unnecessary forwarding from occurring - LOGGER.debug("No need for any forwarding because file list request is fully served"); - return; - } - - } - - // We may need to forward this request on - boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName())); - if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { - // In relay mode - so ask our other peers if they have it - - long requestTime = getArbitraryDataFileListMessage.getRequestTime(); - int requestHops = getArbitraryDataFileListMessage.getRequestHops() + 1; - long totalRequestTime = now - requestTime; - - if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { - // Relay request hasn't timed out yet, so can potentially be rebroadcast - if (requestHops < RELAY_REQUEST_MAX_HOPS) { - // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast - - Message relayGetArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, requestingPeer); - relayGetArbitraryDataFileListMessage.setId(message.getId()); - - LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); - Network.getInstance().broadcast( - broadcastPeer -> - !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null : - broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryDataFileListMessage - ); - - } - else { - // This relay request has reached the maximum number of allowed hops - } - } - else { - // This relay request has timed out - } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index d5bbcfb6..a4034596 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -25,6 +25,8 @@ import java.security.SecureRandom; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; public class ArbitraryDataFileManager extends Thread { @@ -48,7 +50,7 @@ public class ArbitraryDataFileManager extends Thread { /** * List to keep track of any arbitrary data file hash responses */ - public final List arbitraryDataFileHashResponses = Collections.synchronizedList(new ArrayList<>()); + private final List arbitraryDataFileHashResponses = Collections.synchronizedList(new ArrayList<>()); /** * List to keep track of peers potentially available for direct connections, based on recent requests @@ -67,6 +69,7 @@ public class ArbitraryDataFileManager extends Thread { private ArbitraryDataFileManager() { + this.arbitraryDataFileHashResponseScheduler.scheduleAtFixedRate( this::processResponses, 60, 1, TimeUnit.SECONDS); } public static ArbitraryDataFileManager getInstance() { @@ -81,13 +84,6 @@ public class ArbitraryDataFileManager extends Thread { Thread.currentThread().setName("Arbitrary Data File Manager"); try { - // Use a fixed thread pool to execute the arbitrary data file requests - int threadCount = 5; - ExecutorService arbitraryDataFileRequestExecutor = Executors.newFixedThreadPool(threadCount); - for (int i = 0; i < threadCount; i++) { - arbitraryDataFileRequestExecutor.execute(new ArbitraryDataFileRequestThread()); - } - while (!isStopping) { // Nothing to do yet Thread.sleep(1000); @@ -112,7 +108,6 @@ public class ArbitraryDataFileManager extends Thread { final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT; arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp); - arbitraryDataFileHashResponses.removeIf(entry -> entry.getTimestamp() < relayMinimumTimestamp); final long directConnectionInfoMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_DIRECT_CONNECTION_INFO_TIMEOUT; directConnectionInfo.removeIf(entry -> entry.getTimestamp() < directConnectionInfoMinimumTimestamp); @@ -125,8 +120,7 @@ public class ArbitraryDataFileManager extends Thread { // Fetch data files by hash - public boolean fetchArbitraryDataFiles(Repository repository, - Peer peer, + public boolean fetchArbitraryDataFiles(Peer peer, byte[] signature, ArbitraryTransactionData arbitraryTransactionData, List hashes) throws DataException { @@ -151,16 +145,10 @@ public class ArbitraryDataFileManager extends Thread { if (receivedArbitraryDataFile != null) { LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFile.getHash58(), peer, (endTime-startTime)); receivedAtLeastOneFile = true; - - // Remove this hash from arbitraryDataFileHashResponses now that we have received it - arbitraryDataFileHashResponses.remove(hash58); } else { LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime)); - // Remove this hash from arbitraryDataFileHashResponses now that we have failed to receive it - arbitraryDataFileHashResponses.remove(hash58); - // Stop asking for files from this peer break; } @@ -169,10 +157,6 @@ public class ArbitraryDataFileManager extends Thread { LOGGER.trace("Already requesting data file {} for signature {} from peer {}", arbitraryDataFile, Base58.encode(signature), peer); } } - else { - // Remove this hash from arbitraryDataFileHashResponses because we have a local copy - arbitraryDataFileHashResponses.remove(hash58); - } } if (receivedAtLeastOneFile) { @@ -191,6 +175,38 @@ public class ArbitraryDataFileManager extends Thread { return receivedAtLeastOneFile; } + // Lock to synchronize access to the list + private final Object arbitraryDataFileHashResponseLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService arbitraryDataFileHashResponseScheduler = Executors.newScheduledThreadPool(1); + + + public void addResponse( ArbitraryFileListResponseInfo responseInfo ) { + + synchronized (arbitraryDataFileHashResponseLock) { + this.arbitraryDataFileHashResponses.add(responseInfo); + } + } + + private void processResponses() { + try { + List responsesToProcess; + synchronized (arbitraryDataFileHashResponseLock) { + responsesToProcess = new ArrayList<>(arbitraryDataFileHashResponses); + arbitraryDataFileHashResponses.clear(); + } + + if (responsesToProcess.isEmpty()) return; + + Long now = NTP.getTime(); + + ArbitraryDataFileRequestThread.getInstance().processFileHashes(now, responsesToProcess, this); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, Peer requestingPeer, ArbitraryTransactionData arbitraryTransactionData, byte[] signature, byte[] hash, Message originalMessage) throws DataException { ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature); boolean fileAlreadyExists = existingFile.exists(); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java index b8285052..1872898f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -4,127 +4,172 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo; +import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.transaction.ArbitraryTransactionData; -import org.qortal.event.DataMonitorEvent; -import org.qortal.event.EventBus; import org.qortal.network.Peer; +import org.qortal.network.message.MessageType; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.Base58; import org.qortal.utils.NTP; +import java.net.http.HttpResponse; +import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.HashMap; import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import static java.lang.Thread.NORM_PRIORITY; -public class ArbitraryDataFileRequestThread implements Runnable { +public class ArbitraryDataFileRequestThread { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileRequestThread.class); - public ArbitraryDataFileRequestThread() { + private ConcurrentHashMap executorByPeer = new ConcurrentHashMap<>(); + private ArbitraryDataFileRequestThread() { + cleanupExecutorByPeerScheduler.scheduleAtFixedRate(this::cleanupExecutorsByPeer, 1, 1, TimeUnit.MINUTES); } - @Override - public void run() { - Thread.currentThread().setName("Arbitrary Data File Request Thread"); - Thread.currentThread().setPriority(NORM_PRIORITY); + private static ArbitraryDataFileRequestThread instance = null; + + public static ArbitraryDataFileRequestThread getInstance() { + + if( instance == null ) { + instance = new ArbitraryDataFileRequestThread(); + } + + return instance; + } + + private final ScheduledExecutorService cleanupExecutorByPeerScheduler = Executors.newScheduledThreadPool(1); + + private void cleanupExecutorsByPeer() { try { - while (!Controller.isStopping()) { - Long now = NTP.getTime(); - this.processFileHashes(now); - } - } catch (InterruptedException e) { - // Fall-through to exit thread... + this.executorByPeer.forEach((key, value) -> { + if (value instanceof ThreadPoolExecutor) { + ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) value; + if (threadPoolExecutor.getActiveCount() == 0) { + if (this.executorByPeer.computeIfPresent(key, (k, v) -> null) == null) { + LOGGER.info("removed executor: peer = " + key); + } + } + } else { + LOGGER.warn("casting issue in cleanup"); + } + }); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } - - private void processFileHashes(Long now) throws InterruptedException { + public void processFileHashes(Long now, List responseInfos, ArbitraryDataFileManager arbitraryDataFileManager) { if (Controller.isStopping()) { return; } - ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance(); - String signature58 = null; - String hash58 = null; - Peer peer = null; - boolean shouldProcess = false; + List toProcess = new ArrayList<>(responseInfos.size()); - synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) { - if (!arbitraryDataFileManager.arbitraryDataFileHashResponses.isEmpty()) { + Map responseInfoByHash58 = new HashMap<>(responseInfos.size()); + Map signatureBySignature58 = new HashMap<>(toProcess.size()); + Map> responseInfoBySignature58 = new HashMap<>(); - // Sort by lowest number of node hops first - Comparator lowestHopsFirstComparator = - Comparator.comparingInt(ArbitraryFileListResponseInfo::getRequestHops); - arbitraryDataFileManager.arbitraryDataFileHashResponses.sort(lowestHopsFirstComparator); + for( ArbitraryFileListResponseInfo responseInfo : responseInfos) { - Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.iterator(); - while (iterator.hasNext()) { - if (Controller.isStopping()) { - return; - } + if( responseInfo == null ) continue; - ArbitraryFileListResponseInfo responseInfo = (ArbitraryFileListResponseInfo) iterator.next(); - if (responseInfo == null) { - iterator.remove(); - continue; - } - - hash58 = responseInfo.getHash58(); - peer = responseInfo.getPeer(); - signature58 = responseInfo.getSignature58(); - Long timestamp = responseInfo.getTimestamp(); - - if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { - // Ignore - to be deleted - iterator.remove(); - continue; - } - - // Skip if already requesting, but don't remove, as we might want to retry later - if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) { - // Already requesting - leave this attempt for later - continue; - } - - // We want to process this file - shouldProcess = true; - iterator.remove(); - break; - } + if (Controller.isStopping()) { + return; } + + Peer peer = responseInfo.getPeer(); + + // if relay timeout, then move on + if (now - responseInfo.getTimestamp() >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || responseInfo.getSignature58() == null || peer == null) { + continue; + } + + // Skip if already requesting, but don't remove, as we might want to retry later + if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(responseInfo.getHash58())) { + // Already requesting - leave this attempt for later + arbitraryDataFileManager.addResponse(responseInfo); // don't remove -> adding back, beacause it was removed already above + continue; + } + + + byte[] hash = Base58.decode(responseInfo.getHash58()); + byte[] signature = Base58.decode(responseInfo.getSignature58()); + + // check for null + if (signature == null || hash == null || peer == null) { + continue; + } + + // We want to process this file, store and map data to process later + toProcess.add(responseInfo); + responseInfoByHash58.put(responseInfo.getHash58(), responseInfo); + signatureBySignature58.put(responseInfo.getSignature58(), signature); + responseInfoBySignature58 + .computeIfAbsent(responseInfo.getSignature58(), signature58 -> new ArrayList<>()) + .add(responseInfo); } - if (!shouldProcess) { - // Nothing to do - Thread.sleep(1000L); - return; - } + // if there are no signatures, then there is nothing to process and nothing query the database + if( signatureBySignature58.isEmpty() ) return; - byte[] hash = Base58.decode(hash58); - byte[] signature = Base58.decode(signature58); + List arbitraryTransactionDataList = new ArrayList<>(); // Fetch the transaction data try (final Repository repository = RepositoryManager.getRepository()) { - ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); - if (arbitraryTransactionData == null) { - return; - } - - if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) { - return; - } - - LOGGER.trace("Fetching file {} from peer {} via request thread...", hash58, peer); - arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash)); - + arbitraryTransactionDataList.addAll( + ArbitraryTransactionUtils.fetchTransactionDataList(repository, new ArrayList<>(signatureBySignature58.values()))); } catch (DataException e) { - LOGGER.debug("Unable to process file hashes: {}", e.getMessage()); + LOGGER.warn("Unable to fetch transaction data: {}", e.getMessage()); + } + + if( !arbitraryTransactionDataList.isEmpty() ) { + long start = System.currentTimeMillis(); + + for(ArbitraryTransactionData data : arbitraryTransactionDataList ) { + String signature58 = Base58.encode(data.getSignature()); + for( ArbitraryFileListResponseInfo responseInfo : responseInfoBySignature58.get(signature58)) { + Runnable fetcher = () -> arbitraryDataFileFetcher(arbitraryDataFileManager, responseInfo, data); + this.executorByPeer + .computeIfAbsent( + responseInfo.getPeer().toString(), + peer -> Executors.newFixedThreadPool( + Settings.getInstance().getMaxThreadsForMessageType(MessageType.GET_ARBITRARY_DATA_FILE)) + ) + .execute(fetcher); + } + } + long timeLapse = System.currentTimeMillis() - start; } } -} + + private void arbitraryDataFileFetcher(ArbitraryDataFileManager arbitraryDataFileManager, ArbitraryFileListResponseInfo responseInfo, ArbitraryTransactionData arbitraryTransactionData) { + try { + arbitraryDataFileManager.fetchArbitraryDataFiles( + responseInfo.getPeer(), + arbitraryTransactionData.getSignature(), + arbitraryTransactionData, + Arrays.asList(Base58.decode(responseInfo.getHash58())) + ); + } catch (DataException e) { + LOGGER.warn("Unable to process file hashes: {}", e.getMessage()); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java index 993a2b72..d38d329f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java @@ -24,6 +24,11 @@ import org.qortal.utils.Triple; import java.io.IOException; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.qortal.controller.arbitrary.ArbitraryDataFileListManager.*; @@ -61,6 +66,7 @@ public class ArbitraryMetadataManager { private ArbitraryMetadataManager() { + scheduler.scheduleAtFixedRate(this::processNetworkGetArbitraryMetadataMessage, 60, 1, TimeUnit.SECONDS); } public static ArbitraryMetadataManager getInstance() { @@ -371,107 +377,160 @@ public class ArbitraryMetadataManager { } } + // List to collect messages + private final List messageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object lock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + public void onNetworkGetArbitraryMetadataMessage(Peer peer, Message message) { + // Don't respond if QDN is disabled if (!Settings.getInstance().isQdnEnabled()) { return; } - Controller.getInstance().stats.getArbitraryMetadataMessageStats.requests.incrementAndGet(); - - GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) message; - byte[] signature = getArbitraryMetadataMessage.getSignature(); - String signature58 = Base58.encode(signature); - Long now = NTP.getTime(); - Triple newEntry = new Triple<>(signature58, peer, now); - - // If we've seen this request recently, then ignore - if (arbitraryMetadataRequests.putIfAbsent(message.getId(), newEntry) != null) { - LOGGER.debug("Ignoring metadata request from peer {} for signature {}", peer, signature58); - return; - } - - LOGGER.debug("Received metadata request from peer {} for signature {}", peer, signature58); - - ArbitraryTransactionData transactionData = null; - ArbitraryDataFile metadataFile = null; - - try (final Repository repository = RepositoryManager.getRepository()) { - - // Firstly we need to lookup this file on chain to get its metadata hash - transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature); - if (transactionData instanceof ArbitraryTransactionData) { - - // Check if we're even allowed to serve metadata for this transaction - if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) { - - byte[] metadataHash = transactionData.getMetadataHash(); - if (metadataHash != null) { - - // Load metadata file - metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature); - } - } - } - - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while fetching arbitrary metadata for peer %s", peer), e); - } - - // We should only respond if we have the metadata file - if (metadataFile != null && metadataFile.exists()) { - - // We have the metadata file, so update requests map to reflect that we've sent it - newEntry = new Triple<>(null, null, now); - arbitraryMetadataRequests.put(message.getId(), newEntry); - - ArbitraryMetadataMessage arbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, metadataFile); - arbitraryMetadataMessage.setId(message.getId()); - if (!peer.sendMessage(arbitraryMetadataMessage)) { - LOGGER.debug("Couldn't send metadata"); - peer.disconnect("failed to send metadata"); - return; - } - LOGGER.debug("Sent metadata"); - - // Nothing left to do, so return to prevent any unnecessary forwarding from occurring - LOGGER.debug("No need for any forwarding because metadata request is fully served"); - return; - - } - - // We may need to forward this request on - boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName())); - if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { - // In relay mode - so ask our other peers if they have it - - long requestTime = getArbitraryMetadataMessage.getRequestTime(); - int requestHops = getArbitraryMetadataMessage.getRequestHops() + 1; - long totalRequestTime = now - requestTime; - - if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { - // Relay request hasn't timed out yet, so can potentially be rebroadcast - if (requestHops < RELAY_REQUEST_MAX_HOPS) { - // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast - - Message relayGetArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, requestTime, requestHops); - relayGetArbitraryMetadataMessage.setId(message.getId()); - - LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); - Network.getInstance().broadcast( - broadcastPeer -> - !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null : - broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryMetadataMessage); - - } - else { - // This relay request has reached the maximum number of allowed hops - } - } - else { - // This relay request has timed out - } + synchronized (lock) { + messageList.add(new PeerMessage(peer, message)); } } + private void processNetworkGetArbitraryMetadataMessage() { + + try { + List messagesToProcess; + synchronized (lock) { + messagesToProcess = new ArrayList<>(messageList); + messageList.clear(); + } + + Map signatureBySignature58 = new HashMap<>((messagesToProcess.size())); + Map nowBySignature58 = new HashMap<>(messagesToProcess.size()); + Map peerMessageBySignature58 = new HashMap<>(messagesToProcess.size()); + + for( PeerMessage peerMessage : messagesToProcess) { + Controller.getInstance().stats.getArbitraryMetadataMessageStats.requests.incrementAndGet(); + + GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) peerMessage.message; + byte[] signature = getArbitraryMetadataMessage.getSignature(); + String signature58 = Base58.encode(signature); + Long now = NTP.getTime(); + Triple newEntry = new Triple<>(signature58, peerMessage.peer, now); + + // If we've seen this request recently, then ignore + if (arbitraryMetadataRequests.putIfAbsent(peerMessage.message.getId(), newEntry) != null) { + LOGGER.debug("Ignoring metadata request from peer {} for signature {}", peerMessage.peer, signature58); + continue; + } + + LOGGER.debug("Received metadata request from peer {} for signature {}", peerMessage.peer, signature58); + + signatureBySignature58.put(signature58, signature); + nowBySignature58.put(signature58, now); + peerMessageBySignature58.put(signature58, peerMessage); + } + + if( signatureBySignature58.isEmpty() ) return; + + List transactionDataList; + try (final Repository repository = RepositoryManager.getRepository()) { + + // Firstly we need to lookup this file on chain to get its metadata hash + transactionDataList = repository.getTransactionRepository().fromSignatures(new ArrayList(signatureBySignature58.values())); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary transactions"), e); + return; + } + + Map dataBySignature58 + = transactionDataList.stream() + .filter(data -> data instanceof ArbitraryTransactionData) + .map(ArbitraryTransactionData.class::cast) + .collect(Collectors.toMap(data -> Base58.encode(data.getSignature()), Function.identity())); + + for(Map.Entry entry : dataBySignature58.entrySet()) { + String signature58 = entry.getKey(); + ArbitraryTransactionData transactionData = entry.getValue(); + + try { + + // Check if we're even allowed to serve metadata for this transaction + if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) { + + byte[] metadataHash = transactionData.getMetadataHash(); + if (metadataHash != null) { + + // Load metadata file + ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, transactionData.getSignature()); + // We should only respond if we have the metadata file + if (metadataFile != null && metadataFile.exists()) { + + PeerMessage peerMessage = peerMessageBySignature58.get(signature58); + Message message = peerMessage.message; + Peer peer = peerMessage.peer; + + // We have the metadata file, so update requests map to reflect that we've sent it + Triple newEntry = new Triple<>(null, null, nowBySignature58.get(signature58)); + arbitraryMetadataRequests.put(message.getId(), newEntry); + + ArbitraryMetadataMessage arbitraryMetadataMessage = new ArbitraryMetadataMessage(entry.getValue().getSignature(), metadataFile); + arbitraryMetadataMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryMetadataMessage)) { + LOGGER.debug("Couldn't send metadata"); + peer.disconnect("failed to send metadata"); + continue; + } + LOGGER.debug("Sent metadata"); + + // Nothing left to do, so return to prevent any unnecessary forwarding from occurring + LOGGER.debug("No need for any forwarding because metadata request is fully served"); + } + + } + } + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary metadata"), e); + } + + // We may need to forward this request on + boolean isBlocked = (transactionDataList == null || ListUtils.isNameBlocked(transactionData.getName())); + if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { + // In relay mode - so ask our other peers if they have it + + PeerMessage peerMessage = peerMessageBySignature58.get(signature58); + GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) peerMessage.message; + long requestTime = getArbitraryMetadataMessage.getRequestTime(); + int requestHops = getArbitraryMetadataMessage.getRequestHops() + 1; + long totalRequestTime = nowBySignature58.get(signature58) - requestTime; + + if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { + // Relay request hasn't timed out yet, so can potentially be rebroadcast + if (requestHops < RELAY_REQUEST_MAX_HOPS) { + // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast + + byte[] signature = signatureBySignature58.get(signature58); + Message relayGetArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, requestTime, requestHops); + relayGetArbitraryMetadataMessage.setId(getArbitraryMetadataMessage.getId()); + + Peer peer = peerMessage.peer; + LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); + Network.getInstance().broadcast( + broadcastPeer -> + !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null : + broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryMetadataMessage); + + } else { + // This relay request has reached the maximum number of allowed hops + } + } else { + // This relay request has timed out + } + } + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } } diff --git a/src/main/java/org/qortal/controller/arbitrary/PeerMessage.java b/src/main/java/org/qortal/controller/arbitrary/PeerMessage.java new file mode 100644 index 00000000..e77eca4b --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/PeerMessage.java @@ -0,0 +1,22 @@ +package org.qortal.controller.arbitrary; + +import org.qortal.network.Peer; +import org.qortal.network.message.Message; + +public class PeerMessage { + Peer peer; + Message message; + + public PeerMessage(Peer peer, Message message) { + this.peer = peer; + this.message = message; + } + + public Peer getPeer() { + return peer; + } + + public Message getMessage() { + return message; + } +} diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index 654513f2..c17e5758 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -8,6 +8,7 @@ import org.qortal.account.PrivateKeyAccount; import org.qortal.api.model.crosschain.TradeBotCreateRequest; import org.qortal.controller.Controller; import org.qortal.controller.Synchronizer; +import org.qortal.controller.arbitrary.PeerMessage; import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult; import org.qortal.crosschain.*; import org.qortal.crypto.Crypto; @@ -37,7 +38,12 @@ import org.qortal.utils.NTP; import java.awt.TrayIcon.MessageType; import java.security.SecureRandom; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * Performing cross-chain trading steps on behalf of user. @@ -118,6 +124,9 @@ public class TradeBot implements Listener { private Map validTrades = new HashMap<>(); private TradeBot() { + + tradePresenceMessageScheduler.scheduleAtFixedRate( this::processTradePresencesMessages, 60, 1, TimeUnit.SECONDS); + EventBus.INSTANCE.addListener(event -> TradeBot.getInstance().listen(event)); } @@ -551,77 +560,139 @@ public class TradeBot implements Listener { } } + // List to collect messages + private final List tradePresenceMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object tradePresenceMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService tradePresenceMessageScheduler = Executors.newScheduledThreadPool(1); + public void onTradePresencesMessage(Peer peer, Message message) { - TradePresencesMessage tradePresencesMessage = (TradePresencesMessage) message; - List peersTradePresences = tradePresencesMessage.getTradePresences(); + synchronized (tradePresenceMessageLock) { + tradePresenceMessageList.add(new PeerMessage(peer, message)); + } + } - long now = NTP.getTime(); - // Timestamps before this are too far into the past - long pastThreshold = now; - // Timestamps after this are too far into the future - long futureThreshold = now + PRESENCE_LIFETIME; + public void processTradePresencesMessages() { - Map> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap(); + try { + List messagesToProcess; + synchronized (tradePresenceMessageLock) { + messagesToProcess = new ArrayList<>(tradePresenceMessageList); + tradePresenceMessageList.clear(); + } - int newCount = 0; + if( messagesToProcess.isEmpty() ) return; - try (final Repository repository = RepositoryManager.getRepository()) { - for (TradePresenceData peersTradePresence : peersTradePresences) { - long timestamp = peersTradePresence.getTimestamp(); + Map> tradePresencesByPeer = new HashMap<>(messagesToProcess.size()); - // Ignore if timestamp is out of bounds - if (timestamp < pastThreshold || timestamp > futureThreshold) { - if (timestamp < pastThreshold) - LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too old vs {}", - peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold - ); - else - LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too new vs {}", - peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold + // map all trade presences from the messages to their peer + for( PeerMessage peerMessage : messagesToProcess ) { + TradePresencesMessage tradePresencesMessage = (TradePresencesMessage) peerMessage.getMessage(); + + List peersTradePresences = tradePresencesMessage.getTradePresences(); + + tradePresencesByPeer.put(peerMessage.getPeer(), peersTradePresences); + } + + long now = NTP.getTime(); + // Timestamps before this are too far into the past + long pastThreshold = now; + // Timestamps after this are too far into the future + long futureThreshold = now + PRESENCE_LIFETIME; + + Map> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap(); + + int newCount = 0; + + Map> peersByAtAddress = new HashMap<>(tradePresencesByPeer.size()); + Map tradePresenceByAtAddress = new HashMap<>(tradePresencesByPeer.size()); + + // for each batch of trade presence data from a peer, validate and populate the maps declared above + for ( Map.Entry> entry: tradePresencesByPeer.entrySet()) { + + Peer peer = entry.getKey(); + + for( TradePresenceData peersTradePresence : entry.getValue() ) { + // TradePresenceData peersTradePresence + long timestamp = peersTradePresence.getTimestamp(); + + // Ignore if timestamp is out of bounds + if (timestamp < pastThreshold || timestamp > futureThreshold) { + if (timestamp < pastThreshold) + LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too old vs {}", + peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold + ); + else + LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too new vs {}", + peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold + ); + + continue; + } + + ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey()); + + // Ignore if we've previously verified this timestamp+publickey combo or sent timestamp is older + TradePresenceData existingTradeData = this.safeAllTradePresencesByPubkey.get(pubkeyByteArray); + if (existingTradeData != null && timestamp <= existingTradeData.getTimestamp()) { + if (timestamp == existingTradeData.getTimestamp()) + LOGGER.trace("Ignoring trade presence {} from peer {} as we have verified timestamp {} before", + peersTradePresence.getAtAddress(), peer, timestamp + ); + else + LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is older than latest {}", + peersTradePresence.getAtAddress(), peer, timestamp, existingTradeData.getTimestamp() + ); + + continue; + } + + // Check timestamp signature + byte[] timestampSignature = peersTradePresence.getSignature(); + byte[] timestampBytes = Longs.toByteArray(timestamp); + byte[] publicKey = peersTradePresence.getPublicKey(); + if (!Crypto.verify(publicKey, timestampSignature, timestampBytes)) { + LOGGER.trace("Ignoring trade presence {} from peer {} as signature failed to verify", + peersTradePresence.getAtAddress(), peer ); - continue; + continue; + } + + peersByAtAddress.computeIfAbsent(peersTradePresence.getAtAddress(), address -> new ArrayList<>()).add(peer); + tradePresenceByAtAddress.put(peersTradePresence.getAtAddress(), peersTradePresence); } + } - ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey()); + if( tradePresenceByAtAddress.isEmpty() ) return; - // Ignore if we've previously verified this timestamp+publickey combo or sent timestamp is older - TradePresenceData existingTradeData = this.safeAllTradePresencesByPubkey.get(pubkeyByteArray); - if (existingTradeData != null && timestamp <= existingTradeData.getTimestamp()) { - if (timestamp == existingTradeData.getTimestamp()) - LOGGER.trace("Ignoring trade presence {} from peer {} as we have verified timestamp {} before", - peersTradePresence.getAtAddress(), peer, timestamp - ); - else - LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is older than latest {}", - peersTradePresence.getAtAddress(), peer, timestamp, existingTradeData.getTimestamp() - ); + List atDataList; + try (final Repository repository = RepositoryManager.getRepository()) { + atDataList = repository.getATRepository().fromATAddresses( new ArrayList<>(tradePresenceByAtAddress.keySet()) ); + } catch (DataException e) { + LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e); + return; + } - continue; - } + Map> supplierByAtAddress = new HashMap<>(atDataList.size()); - // Check timestamp signature - byte[] timestampSignature = peersTradePresence.getSignature(); - byte[] timestampBytes = Longs.toByteArray(timestamp); - byte[] publicKey = peersTradePresence.getPublicKey(); - if (!Crypto.verify(publicKey, timestampSignature, timestampBytes)) { - LOGGER.trace("Ignoring trade presence {} from peer {} as signature failed to verify", - peersTradePresence.getAtAddress(), peer - ); + List validatedAtDataList = new ArrayList<>(atDataList.size()); - continue; - } + // for each trade + for( ATData atData : atDataList ) { - ATData atData = repository.getATRepository().fromATAddress(peersTradePresence.getAtAddress()); + TradePresenceData peersTradePresence = tradePresenceByAtAddress.get(atData.getATAddress()); if (atData == null || atData.getIsFrozen() || atData.getIsFinished()) { if (atData == null) - LOGGER.trace("Ignoring trade presence {} from peer {} as AT doesn't exist", - peersTradePresence.getAtAddress(), peer + LOGGER.trace("Ignoring trade presence {} from peer as AT doesn't exist", + peersTradePresence.getAtAddress() ); else - LOGGER.trace("Ignoring trade presence {} from peer {} as AT is frozen or finished", - peersTradePresence.getAtAddress(), peer + LOGGER.trace("Ignoring trade presence {} from peer as AT is frozen or finished", + peersTradePresence.getAtAddress() ); continue; @@ -630,51 +701,87 @@ public class TradeBot implements Listener { ByteArray atCodeHash = ByteArray.wrap(atData.getCodeHash()); Supplier acctSupplier = acctSuppliersByCodeHash.get(atCodeHash); if (acctSupplier == null) { - LOGGER.trace("Ignoring trade presence {} from peer {} as AT isn't a known ACCT?", - peersTradePresence.getAtAddress(), peer + LOGGER.trace("Ignoring trade presence {} from peer as AT isn't a known ACCT?", + peersTradePresence.getAtAddress() ); continue; } - - CrossChainTradeData tradeData = acctSupplier.get().populateTradeData(repository, atData); - if (tradeData == null) { - LOGGER.trace("Ignoring trade presence {} from peer {} as trade data not found?", - peersTradePresence.getAtAddress(), peer - ); - - continue; - } - - // Convert signer's public key to address form - String signerAddress = peersTradePresence.getTradeAddress(); - - // Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form) - if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) { - LOGGER.trace("Ignoring trade presence {} from peer {} as signer isn't Alice or Bob?", - peersTradePresence.getAtAddress(), peer - ); - - continue; - } - - // This is new to us - this.allTradePresencesByPubkey.put(pubkeyByteArray, peersTradePresence); - ++newCount; - - LOGGER.trace("Added trade presence {} from peer {} with timestamp {}", - peersTradePresence.getAtAddress(), peer, timestamp - ); - - EventBus.INSTANCE.notify(new TradePresenceEvent(peersTradePresence)); + validatedAtDataList.add(atData); } - } catch (DataException e) { - LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e); - } - if (newCount > 0) { - LOGGER.debug("New trade presences: {}, all trade presences: {}", newCount, allTradePresencesByPubkey.size()); - rebuildSafeAllTradePresences(); + // populated data for each trade + List crossChainTradeDataList; + + // validated trade data grouped by code (cross chain coin) + Map> atDataByCodeHash + = validatedAtDataList.stream().collect( + Collectors.groupingBy(data -> ByteArray.wrap(data.getCodeHash()))); + + try (final Repository repository = RepositoryManager.getRepository()) { + + crossChainTradeDataList = new ArrayList<>(); + + // for each code (cross chain coin), get each trade, then populate trade data + for( Map.Entry> entry : atDataByCodeHash.entrySet() ) { + + Supplier acctSupplier = acctSuppliersByCodeHash.get(entry.getKey()); + + crossChainTradeDataList.addAll( + acctSupplier.get().populateTradeDataList( + repository, + entry.getValue() + ) + .stream().filter( data -> data != null ) + .collect(Collectors.toList()) + ); + } + } catch (DataException e) { + LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e); + return; + } + + // for each populated trade data, validate and fire event + for( CrossChainTradeData tradeData : crossChainTradeDataList ) { + + List peers = peersByAtAddress.get(tradeData.qortalAtAddress); + + for( Peer peer : peers ) { + + TradePresenceData peersTradePresence = tradePresenceByAtAddress.get(tradeData.qortalAtAddress); + + // Convert signer's public key to address form + String signerAddress = peersTradePresence.getTradeAddress(); + + // Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form) + if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) { + LOGGER.trace("Ignoring trade presence {} from peer {} as signer isn't Alice or Bob?", + peersTradePresence.getAtAddress(), peer + ); + + continue; + } + + ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey()); + + // This is new to us + this.allTradePresencesByPubkey.put(pubkeyByteArray, peersTradePresence); + ++newCount; + + LOGGER.trace("Added trade presence {} from peer {} with timestamp {}", + peersTradePresence.getAtAddress(), peer, tradeData.creationTimestamp + ); + + EventBus.INSTANCE.notify(new TradePresenceEvent(peersTradePresence)); + } + } + + if (newCount > 0) { + LOGGER.info("New trade presences: {}, all trade presences: {}", newCount, allTradePresencesByPubkey.size()); + rebuildSafeAllTradePresences(); + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } diff --git a/src/main/java/org/qortal/crosschain/ACCT.java b/src/main/java/org/qortal/crosschain/ACCT.java index de28cfce..83b453e6 100644 --- a/src/main/java/org/qortal/crosschain/ACCT.java +++ b/src/main/java/org/qortal/crosschain/ACCT.java @@ -6,6 +6,9 @@ import org.qortal.data.crosschain.CrossChainTradeData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; +import java.util.List; +import java.util.OptionalLong; + public interface ACCT { public byte[] getCodeBytesHash(); @@ -16,8 +19,12 @@ public interface ACCT { public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException; + public List populateTradeDataList(Repository respository, List atDataList) throws DataException; + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException; + CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException; + public byte[] buildCancelMessage(String creatorQortalAddress); public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException; diff --git a/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java b/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java index cb855466..d7b8bb77 100644 --- a/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java +++ b/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java @@ -4,6 +4,7 @@ import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -19,6 +20,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -608,7 +610,14 @@ public class BitcoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -617,13 +626,14 @@ public class BitcoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -636,8 +646,13 @@ public class BitcoinACCTv1 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/BitcoinACCTv3.java b/src/main/java/org/qortal/crosschain/BitcoinACCTv3.java index ecf768ed..efef0959 100644 --- a/src/main/java/org/qortal/crosschain/BitcoinACCTv3.java +++ b/src/main/java/org/qortal/crosschain/BitcoinACCTv3.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -569,7 +571,14 @@ public class BitcoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -578,13 +587,14 @@ public class BitcoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -597,8 +607,13 @@ public class BitcoinACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/DigibyteACCTv3.java b/src/main/java/org/qortal/crosschain/DigibyteACCTv3.java index 9fa67592..5dec80f0 100644 --- a/src/main/java/org/qortal/crosschain/DigibyteACCTv3.java +++ b/src/main/java/org/qortal/crosschain/DigibyteACCTv3.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -569,7 +571,14 @@ public class DigibyteACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -578,13 +587,14 @@ public class DigibyteACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -597,8 +607,13 @@ public class DigibyteACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java b/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java index a5ec6f1f..a0caeb80 100644 --- a/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java +++ b/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -566,7 +568,14 @@ public class DogecoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -575,13 +584,14 @@ public class DogecoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -594,8 +604,13 @@ public class DogecoinACCTv1 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java b/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java index 06b04705..18581b2c 100644 --- a/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java +++ b/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -569,7 +571,14 @@ public class DogecoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -578,13 +587,14 @@ public class DogecoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -597,8 +607,13 @@ public class DogecoinACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java b/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java index 6a828981..b91e8e65 100644 --- a/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java +++ b/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java @@ -4,6 +4,7 @@ import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -19,6 +20,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -559,7 +561,14 @@ public class LitecoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -568,13 +577,14 @@ public class LitecoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -587,8 +597,13 @@ public class LitecoinACCTv1 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java b/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java index 4a533b4b..97222317 100644 --- a/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java +++ b/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java @@ -4,6 +4,7 @@ import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -19,6 +20,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -562,7 +564,14 @@ public class LitecoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -571,13 +580,14 @@ public class LitecoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -590,8 +600,13 @@ public class LitecoinACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/PirateChain.java b/src/main/java/org/qortal/crosschain/PirateChain.java index 48178f28..53a6fa9c 100644 --- a/src/main/java/org/qortal/crosschain/PirateChain.java +++ b/src/main/java/org/qortal/crosschain/PirateChain.java @@ -52,12 +52,7 @@ public class PirateChain extends Bitcoiny { public Collection getServers() { return Arrays.asList( // Servers chosen on NO BASIS WHATSOEVER from various sources! - new Server("lightd.pirate.black", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr1.qortal.online", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr2.qortal.online", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr3.qortal.online", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr4.qortal.online", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr5.qortal.online", Server.ConnectionType.SSL, 443) + new Server("lightd.pirate.black", Server.ConnectionType.SSL, 443) ); } diff --git a/src/main/java/org/qortal/crosschain/PirateChainACCTv3.java b/src/main/java/org/qortal/crosschain/PirateChainACCTv3.java index 8873eeab..4c2893ea 100644 --- a/src/main/java/org/qortal/crosschain/PirateChainACCTv3.java +++ b/src/main/java/org/qortal/crosschain/PirateChainACCTv3.java @@ -4,6 +4,7 @@ import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -19,6 +20,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -580,7 +582,14 @@ public class PirateChainACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -589,13 +598,14 @@ public class PirateChainACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -608,8 +618,13 @@ public class PirateChainACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/PirateWallet.java b/src/main/java/org/qortal/crosschain/PirateWallet.java index d8fdc351..9ecfc2c0 100644 --- a/src/main/java/org/qortal/crosschain/PirateWallet.java +++ b/src/main/java/org/qortal/crosschain/PirateWallet.java @@ -8,6 +8,7 @@ import org.bouncycastle.util.encoders.DecoderException; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.controller.PirateChainWalletController; import org.qortal.crypto.Crypto; import org.qortal.settings.Settings; @@ -67,8 +68,8 @@ public class PirateWallet { } // Pick a random server - PirateLightClient.Server server = this.getRandomServer(); - String serverUri = String.format("https://%s:%d/", server.hostname, server.port); + ChainableServer server = PirateChain.getInstance().blockchainProvider.getCurrentServer(); + String serverUri = String.format("https://%s:%d/", server.getHostName(), server.getPort()); // Pirate library uses base64 encoding String entropy64 = Base64.toBase64String(this.entropyBytes); diff --git a/src/main/java/org/qortal/crosschain/RavencoinACCTv3.java b/src/main/java/org/qortal/crosschain/RavencoinACCTv3.java index f027e9ca..b880f831 100644 --- a/src/main/java/org/qortal/crosschain/RavencoinACCTv3.java +++ b/src/main/java/org/qortal/crosschain/RavencoinACCTv3.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -569,7 +571,14 @@ public class RavencoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -578,13 +587,14 @@ public class RavencoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -597,8 +607,13 @@ public class RavencoinACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index f500b2e8..3737852f 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -982,7 +982,7 @@ public class Network { if (maxThreadsForMessageType != null) { Integer threadCount = threadsPerMessageType.get(message.getType()); if (threadCount != null && threadCount >= maxThreadsForMessageType) { - LOGGER.trace("Discarding {} message as there are already {} active threads", message.getType().name(), threadCount); + LOGGER.warn("Discarding {} message as there are already {} active threads", message.getType().name(), threadCount); return; } } diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 2b653ab5..7e6c16ff 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -14,6 +14,8 @@ public interface ATRepository { /** Returns ATData using AT's address or null if none found */ public ATData fromATAddress(String atAddress) throws DataException; + public List fromATAddresses(List atAddresses) throws DataException; + /** Returns where AT with passed address exists in repository */ public boolean exists(String atAddress) throws DataException; @@ -62,6 +64,8 @@ public interface ATRepository { */ public ATStateData getLatestATState(String atAddress) throws DataException; + public List getLatestATStates(List collect) throws DataException; + /** * Returns final ATStateData for ATs matching codeHash (required) * and specific data segment value (optional). diff --git a/src/main/java/org/qortal/repository/AccountRepository.java b/src/main/java/org/qortal/repository/AccountRepository.java index f68fe8eb..daac1e02 100644 --- a/src/main/java/org/qortal/repository/AccountRepository.java +++ b/src/main/java/org/qortal/repository/AccountRepository.java @@ -130,6 +130,8 @@ public interface AccountRepository { */ public AccountBalanceData getBalance(String address, long assetId) throws DataException; + public List getBalances(List addresses, long assetId) throws DataException; + /** Returns all account balances for given assetID, optionally excluding zero balances. */ public List getAssetBalances(long assetId, Boolean excludeZero) throws DataException; diff --git a/src/main/java/org/qortal/repository/TransactionRepository.java b/src/main/java/org/qortal/repository/TransactionRepository.java index d4517485..79d55c9a 100644 --- a/src/main/java/org/qortal/repository/TransactionRepository.java +++ b/src/main/java/org/qortal/repository/TransactionRepository.java @@ -18,6 +18,8 @@ public interface TransactionRepository { public TransactionData fromSignature(byte[] signature) throws DataException; + public List fromSignatures(List signatures) throws DataException; + public TransactionData fromReference(byte[] reference) throws DataException; public TransactionData fromHeightAndSequence(int height, int sequence) throws DataException; @@ -351,4 +353,5 @@ public interface TransactionRepository { public void delete(TransactionData transactionData) throws DataException; + } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index 6310ec02..c941e15a 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -15,8 +15,12 @@ import org.qortal.utils.ByteArray; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.qortal.data.account.AccountData; @@ -76,6 +80,63 @@ public class HSQLDBATRepository implements ATRepository { } } + @Override + public List fromATAddresses(List atAddresses) throws DataException { + String sql = "SELECT creator, created_when, version, asset_id, code_bytes, code_hash, " + + "is_sleeping, sleep_until_height, is_finished, had_fatal_error, " + + "is_frozen, frozen_balance, sleep_until_message_timestamp, AT_address " + + "FROM ATs " + + "WHERE AT_address IN (" + + String.join(", ", Collections.nCopies(atAddresses.size(), "?")) + + ")" + ; + + List list; + try (ResultSet resultSet = this.repository.checkedExecute(sql, atAddresses.toArray(new String[atAddresses.size()]))) { + if (resultSet == null) { + return new ArrayList<>(0); + } + + list = new ArrayList<>(atAddresses.size()); + + do { + byte[] creatorPublicKey = resultSet.getBytes(1); + long created = resultSet.getLong(2); + int version = resultSet.getInt(3); + long assetId = resultSet.getLong(4); + byte[] codeBytes = resultSet.getBytes(5); // Actually BLOB + byte[] codeHash = resultSet.getBytes(6); + boolean isSleeping = resultSet.getBoolean(7); + + Integer sleepUntilHeight = resultSet.getInt(8); + if (sleepUntilHeight == 0 && resultSet.wasNull()) + sleepUntilHeight = null; + + boolean isFinished = resultSet.getBoolean(9); + boolean hadFatalError = resultSet.getBoolean(10); + boolean isFrozen = resultSet.getBoolean(11); + + Long frozenBalance = resultSet.getLong(12); + if (frozenBalance == 0 && resultSet.wasNull()) + frozenBalance = null; + + Long sleepUntilMessageTimestamp = resultSet.getLong(13); + if (sleepUntilMessageTimestamp == 0 && resultSet.wasNull()) + sleepUntilMessageTimestamp = null; + + String atAddress = resultSet.getString(14); + + list.add(new ATData(atAddress, creatorPublicKey, created, version, assetId, codeBytes, codeHash, + isSleeping, sleepUntilHeight, isFinished, hadFatalError, isFrozen, frozenBalance, + sleepUntilMessageTimestamp)); + } while ( resultSet.next()); + + return list; + } catch (SQLException e) { + throw new DataException("Unable to fetch AT from repository", e); + } + } + @Override public boolean exists(String atAddress) throws DataException { try { @@ -403,6 +464,56 @@ public class HSQLDBATRepository implements ATRepository { } } + @Override + public List getLatestATStates(List atAddresses) throws DataException{ + String sql = "SELECT height, state_data, state_hash, fees, is_initial, sleep_until_message_timestamp, AT_address " + + "FROM ATStates " + + "JOIN ATStatesData USING (AT_address, height) " + + "WHERE ATStates.AT_address IN (" + + String.join(", ", Collections.nCopies(atAddresses.size(), "?")) + + ")"; + + List stateDataList; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, atAddresses.toArray(new String[atAddresses.size()]))) { + if (resultSet == null) + return new ArrayList<>(0); + + stateDataList = new ArrayList<>(); + + do { + int height = resultSet.getInt(1); + byte[] stateData = resultSet.getBytes(2); // Actually BLOB + byte[] stateHash = resultSet.getBytes(3); + long fees = resultSet.getLong(4); + boolean isInitial = resultSet.getBoolean(5); + + Long sleepUntilMessageTimestamp = resultSet.getLong(6); + if (sleepUntilMessageTimestamp == 0 && resultSet.wasNull()) + sleepUntilMessageTimestamp = null; + + String atAddress = resultSet.getString(7); + stateDataList.add(new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial, sleepUntilMessageTimestamp)); + } while( resultSet.next()); + } catch (SQLException e) { + throw new DataException("Unable to fetch latest AT state from repository", e); + } + + Map> stateDataByAtAddress + = stateDataList.stream() + .collect(Collectors.groupingBy(ATStateData::getATAddress)); + + List latestForEachAtAddress + = stateDataByAtAddress.values().stream() + .map(list -> list.stream() + .max(Comparator.comparing(ATStateData::getHeight)) + .orElse(null)) + .filter(obj -> obj != null) + .collect(Collectors.toList()); + + return latestForEachAtAddress; + } + @Override public List getMatchingFinalATStates(byte[] codeHash, byte[] buyerPublicKey, byte[] sellerPublicKey, Boolean isFinished, Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight, diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java index 9cec85b2..7a0e486c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java @@ -407,6 +407,39 @@ public class HSQLDBAccountRepository implements AccountRepository { } } + @Override + public List getBalances(List addresses, long assetId) throws DataException { + + StringBuffer sql = new StringBuffer(); + sql.append("SELECT balance, account, asset_id FROM AccountBalances "); + sql.append("WHERE account IN ("); + sql.append(String.join(", ", Collections.nCopies(addresses.size(), "?"))); + sql.append(")"); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), addresses.toArray(new String[addresses.size()]))) { + if (resultSet == null) + return new ArrayList<>(0); + + List balances = new ArrayList<>(addresses.size()); + do { + long balance = resultSet.getLong(1); + String address = resultSet.getString(2); + Long assetIdResult = resultSet.getLong(3); + + if( assetIdResult != assetId ) { + LOGGER.warn("assetIdResult = " + assetIdResult); + continue; + } + + balances.add(new AccountBalanceData(address, assetId, balance) ); + } while( resultSet.next()); + + return balances; + } catch (SQLException e) { + throw new DataException("Unable to fetch account balance from repository", e); + } + } + @Override public List getAssetBalances(long assetId, Boolean excludeZero) throws DataException { StringBuilder sql = new StringBuilder(1024); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java index 48262dee..535c3ed6 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java @@ -40,13 +40,25 @@ public class HSQLDBChatRepository implements ChatRepository { StringBuilder sql = new StringBuilder(1024); + String tableName; + + // if the PrimaryTable is available, then use it + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + LOGGER.debug("using PrimaryNames for chat transactions"); + tableName = "PrimaryNames"; + } + else { + LOGGER.debug("using Names for chat transactions"); + tableName = "Names"; + } + sql.append("SELECT created_when, tx_group_id, Transactions.reference, creator, " + "sender, SenderNames.name, recipient, RecipientNames.name, " + "chat_reference, data, is_text, is_encrypted, signature " + "FROM ChatTransactions " + "JOIN Transactions USING (signature) " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " - + "LEFT OUTER JOIN Names AS RecipientNames ON RecipientNames.owner = recipient "); + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS RecipientNames ON RecipientNames.owner = recipient "); // WHERE clauses @@ -152,11 +164,11 @@ public class HSQLDBChatRepository implements ChatRepository { // if the PrimaryTable is available, then use it if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { - LOGGER.info("using PrimaryNames for chat transactions"); + LOGGER.debug("using PrimaryNames for chat transactions"); tableName = "PrimaryNames"; } else { - LOGGER.info("using Names for chat transactions"); + LOGGER.debug("using Names for chat transactions"); tableName = "Names"; } @@ -202,6 +214,18 @@ public class HSQLDBChatRepository implements ChatRepository { } private List getActiveGroupChats(String address, Encoding encoding, Boolean hasChatReference) throws DataException { + String tableName; + + // if the PrimaryTable is available, then use it + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + LOGGER.debug("using PrimaryNames for chat transactions"); + tableName = "PrimaryNames"; + } + else { + LOGGER.debug("using Names for chat transactions"); + tableName = "Names"; + } + // Find groups where address is a member and potential latest message details String groupsSql = "SELECT group_id, group_name, latest_timestamp, sender, sender_name, signature, data " + "FROM GroupMembers " @@ -210,7 +234,7 @@ public class HSQLDBChatRepository implements ChatRepository { + "SELECT created_when AS latest_timestamp, sender, name AS sender_name, signature, data " + "FROM ChatTransactions " + "JOIN Transactions USING (signature) " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " // NOTE: We need to qualify "Groups.group_id" here to avoid "General error" bug in HSQLDB v2.5.0 + "WHERE tx_group_id = Groups.group_id AND type = " + TransactionType.CHAT.value + " "; @@ -254,7 +278,7 @@ public class HSQLDBChatRepository implements ChatRepository { String grouplessSql = "SELECT created_when, sender, SenderNames.name, signature, data " + "FROM ChatTransactions " + "JOIN Transactions USING (signature) " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " + "WHERE tx_group_id = 0 " + "AND recipient IS NULL "; @@ -294,6 +318,18 @@ public class HSQLDBChatRepository implements ChatRepository { } private List getActiveDirectChats(String address, Boolean hasChatReference) throws DataException { + String tableName; + + // if the PrimaryTable is available, then use it + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + LOGGER.debug("using PrimaryNames for chat transactions"); + tableName = "PrimaryNames"; + } + else { + LOGGER.debug("using Names for chat transactions"); + tableName = "Names"; + } + // Find chat messages involving address String directSql = "SELECT other_address, name, latest_timestamp, sender, sender_name " + "FROM (" @@ -307,7 +343,7 @@ public class HSQLDBChatRepository implements ChatRepository { + "SELECT created_when AS latest_timestamp, sender, name AS sender_name " + "FROM ChatTransactions " + "NATURAL JOIN Transactions " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " + "WHERE (sender = other_address AND recipient = ?) " + "OR (sender = ? AND recipient = other_address) "; @@ -323,7 +359,7 @@ public class HSQLDBChatRepository implements ChatRepository { directSql += "ORDER BY created_when DESC " + "LIMIT 1" + ") AS LatestMessages " - + "LEFT OUTER JOIN Names ON owner = other_address"; + + "LEFT OUTER JOIN " + tableName + " ON owner = other_address"; Object[] bindParams = new Object[] { address, address, address, address }; diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index fe0b4d0b..cd646cb9 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -155,6 +155,58 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } } + @Override + public List fromSignatures(List signatures) throws DataException { + StringBuffer sql = new StringBuffer(); + + sql.append("SELECT type, reference, creator, created_when, fee, tx_group_id, block_height, approval_status, approval_height, signature "); + sql.append("FROM Transactions WHERE signature IN ("); + sql.append(String.join(", ", Collections.nCopies(signatures.size(), "?"))); + sql.append(")"); + + List list; + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), signatures.toArray(new byte[0][]))) { + if (resultSet == null) { + return new ArrayList<>(0); + } + + list = new ArrayList<>(signatures.size()); + + do { + TransactionType type = TransactionType.valueOf(resultSet.getInt(1)); + + byte[] reference = resultSet.getBytes(2); + byte[] creatorPublicKey = resultSet.getBytes(3); + long timestamp = resultSet.getLong(4); + + Long fee = resultSet.getLong(5); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(6); + + Integer blockHeight = resultSet.getInt(7); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(8)); + Integer approvalHeight = resultSet.getInt(9); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + byte[] signature = resultSet.getBytes(10); + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + list.add( fromBase(type, baseTransactionData) ); + } while( resultSet.next()); + + return list; + } catch (SQLException e) { + throw new DataException("Unable to fetch transactions from repository", e); + } + } + @Override public TransactionData fromReference(byte[] reference) throws DataException { String sql = "SELECT type, signature, creator, created_when, fee, tx_group_id, block_height, approval_status, approval_height " diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 3e82655b..3123ae96 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -759,14 +759,14 @@ public class Settings { maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE", 5)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA", 5)); maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA_FILE_LIST", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE_LIST", 5)); + maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA_FILE_LIST", 50)); + maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE_LIST", 50)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_SIGNATURES", 5)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_METADATA", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_METADATA", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("GET_TRANSACTION", 10)); - maxThreadsPerMessageType.add(new ThreadLimit("TRANSACTION_SIGNATURES", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("TRADE_PRESENCES", 5)); + maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_METADATA", 50)); + maxThreadsPerMessageType.add(new ThreadLimit("GET_TRANSACTION", 50)); + maxThreadsPerMessageType.add(new ThreadLimit("TRANSACTION_SIGNATURES", 50)); + maxThreadsPerMessageType.add(new ThreadLimit("TRADE_PRESENCES", 50)); } // Getters / setters diff --git a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java index 17c966fe..2ebd3b0e 100644 --- a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java @@ -131,16 +131,12 @@ public class ArbitraryIndexUtils { ) ); - LOGGER.info("processed indices by term: count = " + indicesByTerm.size()); - // lock, clear old, load new synchronized( IndexCache.getInstance().getIndicesByTerm() ) { IndexCache.getInstance().getIndicesByTerm().clear(); IndexCache.getInstance().getIndicesByTerm().putAll(indicesByTerm); } - LOGGER.info("loaded indices by term"); - LOGGER.debug("processing indices by issuer ..."); Map> indicesByIssuer = indexDetails.stream().collect( @@ -154,15 +150,11 @@ public class ArbitraryIndexUtils { ) ); - LOGGER.info("processed indices by issuer: count = " + indicesByIssuer.size()); - // lock, clear old, load new synchronized( IndexCache.getInstance().getIndicesByIssuer() ) { IndexCache.getInstance().getIndicesByIssuer().clear(); IndexCache.getInstance().getIndicesByIssuer().putAll(indicesByIssuer); } - - LOGGER.info("loaded indices by issuer"); } } @@ -221,7 +213,6 @@ public class ArbitraryIndexUtils { } } } - Thread.sleep(3000L); } java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index c860a034..1d4cab18 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -48,6 +48,24 @@ public class ArbitraryTransactionUtils { } } + public static List fetchTransactionDataList(final Repository repository, final List signature) { + try { + List transactions = repository.getTransactionRepository().fromSignatures(signature); + + List list + = transactions.stream() + .filter( transaction -> transaction instanceof ArbitraryTransactionData ) + .map( transactionData -> (ArbitraryTransactionData) transactionData) + .collect(Collectors.toList()); + + return list; + + } catch (DataException e) { + LOGGER.error("Repository issue when fetching arbitrary transaction data", e); + return null; + } + } + public static ArbitraryTransactionData fetchLatestPut(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { if (arbitraryTransactionData == null) { return null; diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 14ba924b..2f347d18 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -120,7 +120,8 @@ "nullGroupMembershipHeight": 2012800, "ignoreLevelForRewardShareHeight": 2012800, "adminQueryFixHeight": 2012800, - "multipleNamesPerAccountHeight": 9999999 + "multipleNamesPerAccountHeight": 9999999, + "mintedBlocksAdjustmentRemovalHeight": 9999999 }, "checkpoints": [ { "height": 1136300, "signature": "3BbwawEF2uN8Ni5ofpJXkukoU8ctAPxYoFB7whq9pKfBnjfZcpfEJT4R95NvBDoTP8WDyWvsUvbfHbcr9qSZuYpSKZjUQTvdFf6eqznHGEwhZApWfvXu6zjGCxYCp65F4jsVYYJjkzbjmkCg5WAwN5voudngA23kMK6PpTNygapCzXt" } diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index e410aae4..3bf89ab5 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -115,7 +115,8 @@ "ignoreLevelForRewardShareHeight": 9999999999999, "nullGroupMembershipHeight": 20, "adminQueryFixHeight": 9999999999999, - "multipleNamesPerAccountHeight": 10 + "multipleNamesPerAccountHeight": 10, + "mintedBlocksAdjustmentRemovalHeight": 9999999999999 }, "genesisInfo": { "version": 4,